diff --git a/.backportrc.json b/.backportrc.json
index d2e92817c026b..03f3f892f9227 100644
--- a/.backportrc.json
+++ b/.backportrc.json
@@ -1,10 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.16.0$" : "8.x",
+    "^v8.17.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index a886220c84cda..0ece129a3c238 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -16,10 +16,10 @@ export COMPOSE_HTTP_TIMEOUT
 JOB_BRANCH="$BUILDKITE_BRANCH"
 export JOB_BRANCH
 
-GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+GRADLEW="./gradlew --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
 export GRADLEW
 
-GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+GRADLEW_BAT="./gradlew.bat --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
 export GRADLEW_BAT
 
 export $(cat .ci/java-versions.properties | grep '=' | xargs)
diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml
index f530f237113a9..57412bbe908bc 100644
--- a/.buildkite/pipelines/intake.template.yml
+++ b/.buildkite/pipelines/intake.template.yml
@@ -75,6 +75,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
+    branches: "main 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
index 1bb13c4c10966..37ea49e3a6d95 100644
--- a/.buildkite/pipelines/intake.yml
+++ b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.buildkite/pipelines/lucene-snapshot/run-tests.yml b/.buildkite/pipelines/lucene-snapshot/run-tests.yml
index c76c54a56494e..f7293e051467c 100644
--- a/.buildkite/pipelines/lucene-snapshot/run-tests.yml
+++ b/.buildkite/pipelines/lucene-snapshot/run-tests.yml
@@ -56,7 +56,6 @@ steps:
         matrix:
           setup:
             BWC_VERSION:
-              - 7.17.13
               - 8.9.1
               - 8.10.0
         agents:
diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
index b29747c60617e..8819a5f7f493f 100644
--- a/.buildkite/pipelines/periodic-packaging.yml
+++ b/.buildkite/pipelines/periodic-packaging.yml
@@ -272,8 +272,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.3
+      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -286,7 +286,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0
@@ -304,6 +304,22 @@ steps:
         env:
           BWC_VERSION: 8.16.0
 
+      - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.17.0
+
       - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
         timeout_in_minutes: 300
diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
index cbca7f820c7b7..7b6a6ea72fe83 100644
--- a/.buildkite/pipelines/periodic.yml
+++ b/.buildkite/pipelines/periodic.yml
@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.3#bwcTest
+      - label: 8.15.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -325,6 +325,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.17.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.17.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
         timeout_in_minutes: 300
@@ -410,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -452,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -554,7 +573,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15
diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml
index e94baac8d9448..04ccc41891b3b 100644
--- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml
+++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml
@@ -5,7 +5,7 @@ steps:
     steps:
       - label: "{{matrix.image}} / docker / packaging-tests-unix"
         key: "packaging-tests-unix-docker"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess
         timeout_in_minutes: 300
         matrix:
           setup:
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
index 235a4b2dbb4ad..ea4f34bcbe11e 100644
--- a/.buildkite/pull-requests.json
+++ b/.buildkite/pull-requests.json
@@ -8,6 +8,7 @@
         "admin",
         "write"
       ],
+      "allowed_list": ["elastic-renovate-prod[bot]"],
       "set_commit_status": false,
       "build_on_commit": true,
       "build_on_comment": true,
diff --git a/.buildkite/scripts/dra-workflow.sh b/.buildkite/scripts/dra-workflow.sh
index ecfb8088072a0..81b8225e443a4 100755
--- a/.buildkite/scripts/dra-workflow.sh
+++ b/.buildkite/scripts/dra-workflow.sh
@@ -22,6 +22,7 @@ if [[ "$BRANCH" == "main" ]]; then
 fi
 
 ES_VERSION=$(grep elasticsearch build-tools-internal/version.properties | sed "s/elasticsearch *= *//g")
+echo "ES_VERSION=$ES_VERSION"
 
 VERSION_SUFFIX=""
 if [[ "$WORKFLOW" == "snapshot" ]]; then
@@ -29,7 +30,10 @@ if [[ "$WORKFLOW" == "snapshot" ]]; then
 fi
 
 BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
+echo "BEATS_BUILD_ID=$BEATS_BUILD_ID"
+
 ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
+echo "ML_CPP_BUILD_ID=$ML_CPP_BUILD_ID"
 
 LICENSE_KEY_ARG=""
 BUILD_SNAPSHOT_ARG=""
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index de0505c61a251..2e77631450825 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -14,6 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"
diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh
index bb7547933b213..4d84eded8a3ff 100755
--- a/.ci/scripts/packaging-test.sh
+++ b/.ci/scripts/packaging-test.sh
@@ -78,5 +78,5 @@ sudo -E env \
   --unset=JAVA_HOME \
   SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \
   DOCKER_CONFIG="${HOME}/.docker" \
-  ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
+  ./gradlew -g $HOME/.gradle --console=plain --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
 
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
index 24f58abc72493..c6edc709a8ceb 100644
--- a/.ci/snapshotBwcVersions
+++ b/.ci/snapshotBwcVersions
@@ -1,4 +1,5 @@
 BWC_VERSION:
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 5b98444c044d2..540da14402192 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -39,7 +39,6 @@ gradle @elastic/es-delivery
 build-conventions @elastic/es-delivery
 build-tools @elastic/es-delivery
 build-tools-internal @elastic/es-delivery
-*.gradle @elastic/es-delivery
 .buildkite @elastic/es-delivery
 .ci @elastic/es-delivery
 .idea @elastic/es-delivery
diff --git a/README.asciidoc b/README.asciidoc
index 8d3c96c659896..bac6d0ed71752 100644
--- a/README.asciidoc
+++ b/README.asciidoc
@@ -56,8 +56,8 @@ Quickly set up Elasticsearch and Kibana in Docker for local development or testi
 - If you're using Microsoft Windows, then install https://learn.microsoft.com/en-us/windows/wsl/install[Windows Subsystem for Linux (WSL)].
 
 ==== Trial license
+This setup comes with a one-month trial license that includes all Elastic features.
 
-This setup comes with a one-month trial of the Elastic *Platinum* license.
 After the trial period, the license reverts to *Free and open - Basic*.
 Refer to https://www.elastic.co/subscriptions[Elastic subscriptions] for more information.
 
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java
index 27f4d68b0bc3f..652defa7b39cd 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java
@@ -30,10 +30,13 @@
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.AggregationOperator;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -78,7 +81,10 @@ public class AggregatorBenchmark {
     private static final String DOUBLES = "doubles";
     private static final String BOOLEANS = "booleans";
     private static final String BYTES_REFS = "bytes_refs";
+    private static final String ORDINALS = "ordinals";
     private static final String TWO_LONGS = "two_" + LONGS;
+    private static final String TWO_BYTES_REFS = "two_" + BYTES_REFS;
+    private static final String TWO_ORDINALS = "two_" + ORDINALS;
     private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS;
     private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS;
 
@@ -119,7 +125,21 @@ public class AggregatorBenchmark {
         }
     }
 
-    @Param({ NONE, LONGS, INTS, DOUBLES, BOOLEANS, BYTES_REFS, TWO_LONGS, LONGS_AND_BYTES_REFS, TWO_LONGS_AND_BYTES_REFS })
+    @Param(
+        {
+            NONE,
+            LONGS,
+            INTS,
+            DOUBLES,
+            BOOLEANS,
+            BYTES_REFS,
+            ORDINALS,
+            TWO_LONGS,
+            TWO_BYTES_REFS,
+            TWO_ORDINALS,
+            LONGS_AND_BYTES_REFS,
+            TWO_LONGS_AND_BYTES_REFS }
+    )
     public String grouping;
 
     @Param({ COUNT, COUNT_DISTINCT, MIN, MAX, SUM })
@@ -144,8 +164,12 @@ private static Operator operator(DriverContext driverContext, String grouping, S
             case INTS -> List.of(new BlockHash.GroupSpec(0, ElementType.INT));
             case DOUBLES -> List.of(new BlockHash.GroupSpec(0, ElementType.DOUBLE));
             case BOOLEANS -> List.of(new BlockHash.GroupSpec(0, ElementType.BOOLEAN));
-            case BYTES_REFS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF));
+            case BYTES_REFS, ORDINALS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF));
             case TWO_LONGS -> List.of(new BlockHash.GroupSpec(0, ElementType.LONG), new BlockHash.GroupSpec(1, ElementType.LONG));
+            case TWO_BYTES_REFS, TWO_ORDINALS -> List.of(
+                new BlockHash.GroupSpec(0, ElementType.BYTES_REF),
+                new BlockHash.GroupSpec(1, ElementType.BYTES_REF)
+            );
             case LONGS_AND_BYTES_REFS -> List.of(
                 new BlockHash.GroupSpec(0, ElementType.LONG),
                 new BlockHash.GroupSpec(1, ElementType.BYTES_REF)
@@ -218,6 +242,10 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
                 checkGroupingBlock(prefix, LONGS, page.getBlock(0));
                 checkGroupingBlock(prefix, LONGS, page.getBlock(1));
             }
+            case TWO_BYTES_REFS, TWO_ORDINALS -> {
+                checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(0));
+                checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1));
+            }
             case LONGS_AND_BYTES_REFS -> {
                 checkGroupingBlock(prefix, LONGS, page.getBlock(0));
                 checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1));
@@ -379,7 +407,7 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo
                     throw new AssertionError(prefix + "bad group expected [true] but was [" + groups.getBoolean(1) + "]");
                 }
             }
-            case BYTES_REFS -> {
+            case BYTES_REFS, ORDINALS -> {
                 BytesRefBlock groups = (BytesRefBlock) block;
                 for (int g = 0; g < GROUPS; g++) {
                     if (false == groups.getBytesRef(g, new BytesRef()).equals(bytesGroup(g))) {
@@ -508,6 +536,8 @@ private static Block dataBlock(BlockFactory blockFactory, String blockType) {
     private static List<Block> groupingBlocks(String grouping, String blockType) {
         return switch (grouping) {
             case TWO_LONGS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(LONGS, blockType));
+            case TWO_BYTES_REFS -> List.of(groupingBlock(BYTES_REFS, blockType), groupingBlock(BYTES_REFS, blockType));
+            case TWO_ORDINALS -> List.of(groupingBlock(ORDINALS, blockType), groupingBlock(ORDINALS, blockType));
             case LONGS_AND_BYTES_REFS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(BYTES_REFS, blockType));
             case TWO_LONGS_AND_BYTES_REFS -> List.of(
                 groupingBlock(LONGS, blockType),
@@ -570,6 +600,19 @@ private static Block groupingBlock(String grouping, String blockType) {
                 }
                 yield builder.build();
             }
+            case ORDINALS -> {
+                IntVector.Builder ordinals = blockFactory.newIntVectorBuilder(BLOCK_LENGTH * valuesPerGroup);
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    for (int v = 0; v < valuesPerGroup; v++) {
+                        ordinals.appendInt(i % GROUPS);
+                    }
+                }
+                BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH * valuesPerGroup);
+                for (int i = 0; i < GROUPS; i++) {
+                    bytes.appendBytesRef(bytesGroup(i));
+                }
+                yield new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock();
+            }
             default -> throw new UnsupportedOperationException("unsupported grouping [" + grouping + "]");
         };
     }
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java
index 58b1d2455a7a6..53723f05728b5 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java
@@ -21,7 +21,7 @@ public abstract class AbstractDocValuesForUtilBenchmark {
     protected final int blockSize;
 
     public AbstractDocValuesForUtilBenchmark() {
-        this.forUtil = new DocValuesForUtil();
+        this.forUtil = new DocValuesForUtil(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
         this.blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
     }
 
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java
index b8f0a11e21c8f..284324b3d9206 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.store.DataOutput;
-import org.elasticsearch.index.codec.tsdb.DocValuesForUtil;
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.io.IOException;
@@ -44,7 +43,7 @@ public void setupInvocation(int bitsPerValue) {
 
     @Override
     public void benchmark(int bitsPerValue, Blackhole bh) throws IOException {
-        DocValuesForUtil.decode(bitsPerValue, this.dataInput, this.output);
+        forUtil.decode(bitsPerValue, this.dataInput, this.output);
         bh.consume(this.output);
     }
 }
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java
index 569e8909e1e12..b294fe97c7e7c 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java
@@ -19,7 +19,7 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.core.IOUtils;
@@ -217,19 +217,17 @@ public float squareDistanceScalar() {
         return 1 / (1f + adjustedDistance);
     }
 
-    RandomAccessQuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
+    QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
         var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
         var slice = in.slice("values", 0, in.length());
         return new OffHeapQuantizedByteVectorValues.DenseOffHeapVectorValues(dims, size, sq, false, sim, null, slice);
     }
 
-    RandomVectorScorerSupplier luceneScoreSupplier(RandomAccessQuantizedByteVectorValues values, VectorSimilarityFunction sim)
-        throws IOException {
+    RandomVectorScorerSupplier luceneScoreSupplier(QuantizedByteVectorValues values, VectorSimilarityFunction sim) throws IOException {
         return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorerSupplier(sim, values);
     }
 
-    RandomVectorScorer luceneScorer(RandomAccessQuantizedByteVectorValues values, VectorSimilarityFunction sim, float[] queryVec)
-        throws IOException {
+    RandomVectorScorer luceneScorer(QuantizedByteVectorValues values, VectorSimilarityFunction sim, float[] queryVec) throws IOException {
         return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorer(sim, values, queryVec);
     }
 
diff --git a/branches.json b/branches.json
index e464d6179f2ba..e81d511a88458 100644
--- a/branches.json
+++ b/branches.json
@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.16"
+    },
     {
       "branch": "8.x"
     },
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
index 95f279bfa5162..0535f0bdc3cc8 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -21,25 +21,17 @@ public enum DockerBase {
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
 
-    // Base image with extras for Cloud
-    CLOUD("ubuntu:20.04", "-cloud", "apt-get"),
-
-    // Based on CLOUD above, with more extras. We don't set a base image because
-    // we programmatically extend from the Cloud image.
-    CLOUD_ESS(null, "-cloud-ess", "apt-get"),
-
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bf163e1977002301f7b9fd28fe6837a8cb2dd5c83e4cd45fb67fb28d15d5d40f",
         "-wolfi",
         "apk"
     ),
     // spotless:on
-
     // Based on WOLFI above, with more extras. We don't set a base image because
-    // we programmatically extend from the Wolfi image.
-    WOLFI_ESS(null, "-wolfi-ess", "apk");
+    // we programmatically extend from the wolfi image.
+    CLOUD_ESS(null, "-cloud-ess", "apk");
 
     private final String image;
     private final String suffix;
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
index 25ad5bcf89581..7d9537feaea56 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -15,6 +15,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -61,7 +62,7 @@ public void apply(Project target) {
             ? System.getenv("BUILD_NUMBER")
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
-        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
+        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false && OS.current() != OS.WINDOWS) {
             File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
index 6b93ea10283ae..19309fe2da8a3 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
@@ -181,9 +181,6 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
         if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) {
             return projectName + "wolfi-docker" + archString + "-export";
         }
-        if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS) {
-            return projectName + "wolfi-ess-docker" + archString + "-export";
-        }
         return projectName + distribution.getType().getName();
     }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
index 077a47041861f..ba0e76b3f5b99 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
@@ -22,7 +22,6 @@ public class InternalElasticsearchDistributionTypes {
     public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType();
 
     public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
@@ -32,7 +31,6 @@ public class InternalElasticsearchDistributionTypes {
         DOCKER_IRONBANK,
         DOCKER_CLOUD,
         DOCKER_CLOUD_ESS,
-        DOCKER_WOLFI,
-        DOCKER_WOLFI_ESS
+        DOCKER_WOLFI
     );
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
index cc852e615726a..77ab9557eac33 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
@@ -54,7 +54,6 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
-import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
 
 /**
@@ -153,7 +152,6 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
         lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
-        lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));
diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
index 58ccf69406ff2..5388f942be8d7 100644
--- a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
+++ b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
@@ -59,10 +59,6 @@ org.apache.lucene.util.Version#parseLeniently(java.lang.String)
 
 org.apache.lucene.index.NoMergePolicy#INSTANCE @ explicit use of NoMergePolicy risks forgetting to configure NoMergeScheduler; use org.elasticsearch.common.lucene.Lucene#indexWriterConfigWithNoMerging() instead.
 
-@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead
-org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
-org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()
-
 @defaultMessage Don't interrupt threads use FutureUtils#cancel(Future<T>) instead
 java.util.concurrent.Future#cancel(boolean)
 
diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
index 169c187ef115a..6bc3c2ad4d253 100644
--- a/build-tools-internal/version.properties
+++ b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 9.0.0
-lucene            = 9.12.0
+lucene            = 10.0.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java b/build-tools/src/main/java/org/elasticsearch/gradle/util/PlatformUtils.java
similarity index 53%
rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java
rename to build-tools/src/main/java/org/elasticsearch/gradle/util/PlatformUtils.java
index 550c43d43a536..2f093a19032c8 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/util/PlatformUtils.java
@@ -7,21 +7,17 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.gradle.internal.distribution;
+package org.elasticsearch.gradle.util;
 
-import org.elasticsearch.gradle.ElasticsearchDistributionType;
+import java.util.stream.Collectors;
 
-public class DockerWolfiEssElasticsearchDistributionType implements ElasticsearchDistributionType {
+public class PlatformUtils {
 
-    DockerWolfiEssElasticsearchDistributionType() {}
-
-    @Override
-    public String getName() {
-        return "dockerWolfiEss";
-    }
-
-    @Override
-    public boolean isDocker() {
-        return true;
+    public static String normalize(String input) {
+        return input.lines()
+            .map(it -> it.replace('\\', '/'))
+            .map(it -> it.replaceAll("\\d+\\.\\d\\ds", "0.00s"))
+            .map(it -> it.replace("file:/./", "file:./"))
+            .collect(Collectors.joining("\n"));
     }
 }
diff --git a/distribution/docker/README.md b/distribution/docker/README.md
index 28e6ff314d91a..49facab461edc 100644
--- a/distribution/docker/README.md
+++ b/distribution/docker/README.md
@@ -7,7 +7,7 @@ the [DockerBase] enum.
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
    * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
-   * Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins
+   * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins
      that the ES build generates in an archive directory. It also sets an
      environment variable that points at this directory. This allows plugins to
      be installed from the archive instead of the internet, speeding up
@@ -23,7 +23,6 @@ the [DockerBase] enum.
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
      another UBI build, this time on the regular UBI image, with extra
      hardening. See below for more details.
-
    * Cloud - this is mostly the same as the default image, with some notable differences:
       * `filebeat` and `metricbeat` are included
       * `wget` is included
@@ -31,12 +30,6 @@ the [DockerBase] enum.
         `/app/elasticsearch.sh`. In normal use this file would be bind-mounted
         in, but the image ships a stub version of this file so that the image
         can still be tested.
-   * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins
-     that the ES build generates in an archive directory. It also sets an
-     environment variable that points at this directory. This allows plugins to
-     be installed from the archive instead of the internet, speeding up
-     deployment times.
-
 The long-term goal is for both Cloud images to be retired in favour of the
 default image.
 
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index 99c482d91085a..788e836f8f045 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -288,20 +288,6 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         }
       }
 
-      if (base == DockerBase.CLOUD) {
-        // If we're performing a release build, but `build.id` hasn't been set, we can
-        // infer that we're not at the Docker building stage of the build, and therefore
-        // we should skip the beats part of the build.
-        String buildId = providers.systemProperty('build.id').getOrNull()
-        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
-
-        if (includeBeats) {
-          from configurations.getByName("filebeat_${architecture.classifier}")
-          from configurations.getByName("metricbeat_${architecture.classifier}")
-        }
-        // For some reason, the artifact name can differ depending on what repository we used.
-        rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
-      }
       Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
         project.gradle.sharedServices,
         DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
@@ -381,7 +367,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
   String image = "elasticsearch${base.suffix}"
 
   String namespace = 'elasticsearch'
-  if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
+  if (base == base == DockerBase.CLOUD_ESS) {
     namespace += '-ci'
   }
 
@@ -439,14 +425,15 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
 
     }
 
-  if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD && base != DockerBase.CLOUD_ESS) {
+  if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD_ESS) {
     tasks.named("assemble").configure {
       dependsOn(buildDockerImageTask)
     }
   }
 }
 
-void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) {
+void addBuildEssDockerImageTask(Architecture architecture) {
+  DockerBase dockerBase = DockerBase.CLOUD_ESS
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
   String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
 
@@ -460,22 +447,20 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
         from configurations.allPlugins
       }
 
-      if (dockerBase == DockerBase.WOLFI_ESS) {
-        // If we're performing a release build, but `build.id` hasn't been set, we can
-        // infer that we're not at the Docker building stage of the build, and therefore
-        // we should skip the beats part of the build.
-        String buildId = providers.systemProperty('build.id').getOrNull()
-        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
+      // If we're performing a release build, but `build.id` hasn't been set, we can
+      // infer that we're not at the Docker building stage of the build, and therefore
+      // we should skip the beats part of the build.
+      String buildId = providers.systemProperty('build.id').getOrNull()
+      boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
 
-        if (includeBeats) {
-          from configurations.getByName("filebeat_${architecture.classifier}")
-          from configurations.getByName("metricbeat_${architecture.classifier}")
-        }
-        // For some reason, the artifact name can differ depending on what repository we used.
-        rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
+      if (includeBeats) {
+        from configurations.getByName("filebeat_${architecture.classifier}")
+        from configurations.getByName("metricbeat_${architecture.classifier}")
       }
+      // For some reason, the artifact name can differ depending on what repository we used.
+      rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
 
-      String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix
+      String baseSuffix = DockerBase.WOLFI.suffix
       from(projectDir.resolve("src/docker/Dockerfile.ess")) {
         expand(
           [
@@ -493,7 +478,7 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
     tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) {
 
-      DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI
+      DockerBase base = DockerBase.WOLFI
 
       TaskProvider<DockerBuildTask> buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage"))
       inputs.files(buildBaseTask)
@@ -519,7 +504,7 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
 
 for (final Architecture architecture : Architecture.values()) {
   for (final DockerBase base : DockerBase.values()) {
-    if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
+    if (base == DockerBase.CLOUD_ESS) {
       continue
     }
     addBuildDockerContextTask(architecture, base)
@@ -527,8 +512,7 @@ for (final Architecture architecture : Architecture.values()) {
     addBuildDockerImageTask(architecture, base)
   }
 
-  addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS)
-  addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS)
+  addBuildEssDockerImageTask(architecture)
 }
 
 def exportDockerImages = tasks.register("exportDockerImages")
@@ -550,10 +534,6 @@ subprojects { Project subProject ->
       base = DockerBase.IRON_BANK
     } else if (subProject.name.contains('cloud-ess-')) {
       base = DockerBase.CLOUD_ESS
-    } else if (subProject.name.contains('cloud-')) {
-      base = DockerBase.CLOUD
-    } else if (subProject.name.contains('wolfi-ess')) {
-      base = DockerBase.WOLFI_ESS
     } else if (subProject.name.contains('wolfi-')) {
       base = DockerBase.WOLFI
     }
@@ -561,11 +541,9 @@ subprojects { Project subProject ->
     final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
     final String extension = base == DockerBase.UBI ? 'ubi.tar' :
       (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
-        (base == DockerBase.CLOUD ? 'cloud.tar' :
           (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
             (base == DockerBase.WOLFI ? 'wolfi.tar' :
-              (base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' :
-                'docker.tar')))))
+                'docker.tar')))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')
diff --git a/distribution/docker/cloud-docker-aarch64-export/build.gradle b/distribution/docker/cloud-docker-aarch64-export/build.gradle
deleted file mode 100644
index 537b5a093683e..0000000000000
--- a/distribution/docker/cloud-docker-aarch64-export/build.gradle
+++ /dev/null
@@ -1,2 +0,0 @@
-// This file is intentionally blank. All configuration of the
-// export is done in the parent project.
diff --git a/distribution/docker/cloud-docker-export/build.gradle b/distribution/docker/cloud-docker-export/build.gradle
deleted file mode 100644
index 537b5a093683e..0000000000000
--- a/distribution/docker/cloud-docker-export/build.gradle
+++ /dev/null
@@ -1,2 +0,0 @@
-// This file is intentionally blank. All configuration of the
-// export is done in the parent project.
diff --git a/distribution/docker/src/docker/Dockerfile.ess b/distribution/docker/src/docker/Dockerfile.ess
index 3ca5e8f2b42a3..197af28b93455 100644
--- a/distribution/docker/src/docker/Dockerfile.ess
+++ b/distribution/docker/src/docker/Dockerfile.ess
@@ -2,26 +2,24 @@ FROM ${base_image} AS builder
 
 USER root
 
-<% if (docker_base == "wolfi_ess") { %>
-    # Add plugins infrastructure
-    RUN mkdir -p /opt/plugins/archive
-    RUN chmod -R 0555 /opt/plugins
-
-    COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
-    RUN set -eux ; \\
-        for beat in filebeat metricbeat ; do \\
-          if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
-            echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
-            exit 1 ; \\
-          fi ; \\
-          if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
-            echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
-            exit 1 ; \\
-          fi ; \\
-          mkdir -p /opt/\$beat ; \\
-          tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
-        done
-<% } %>
+# Add plugins infrastructure
+RUN mkdir -p /opt/plugins/archive
+RUN chmod -R 0555 /opt/plugins
+
+COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
+RUN set -eux ; \\
+    for beat in filebeat metricbeat ; do \\
+      if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      mkdir -p /opt/\$beat ; \\
+      tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
+    done
 
 COPY plugins/*.zip /opt/plugins/archive/
 
@@ -29,7 +27,6 @@ RUN chown 1000:1000 /opt/plugins/archive/*
 RUN chmod 0444 /opt/plugins/archive/*
 
 FROM ${base_image}
-<% if (docker_base == "wolfi_ess") { %>
 USER root
 
 RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %>
@@ -44,8 +41,4 @@ RUN mkdir /app && \\
 
 COPY --from=builder --chown=0:0 /opt /opt
 USER 1000:0
-<% } else { %>
-COPY --from=builder /opt/plugins /opt/plugins
-<% } %>
-
 ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive
diff --git a/distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle b/distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle
deleted file mode 100644
index 537b5a093683e..0000000000000
--- a/distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle
+++ /dev/null
@@ -1,2 +0,0 @@
-// This file is intentionally blank. All configuration of the
-// export is done in the parent project.
diff --git a/distribution/docker/wolfi-ess-docker-export/build.gradle b/distribution/docker/wolfi-ess-docker-export/build.gradle
deleted file mode 100644
index 537b5a093683e..0000000000000
--- a/distribution/docker/wolfi-ess-docker-export/build.gradle
+++ /dev/null
@@ -1,2 +0,0 @@
-// This file is intentionally blank. All configuration of the
-// export is done in the parent project.
diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index a523c3ec85ba1..f55d90933ed61 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -62,6 +62,9 @@
 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache
 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached
 
+# Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead
+-Dorg.apache.lucene.store.defaultReadAdvice=normal
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps
diff --git a/distribution/tools/entitlement-agent/impl/build.gradle b/distribution/tools/entitlement-agent/impl/build.gradle
index f73e21505d483..16f134bf0e693 100644
--- a/distribution/tools/entitlement-agent/impl/build.gradle
+++ b/distribution/tools/entitlement-agent/impl/build.gradle
@@ -12,6 +12,13 @@ apply plugin: 'elasticsearch.build'
 dependencies {
   compileOnly project(':distribution:tools:entitlement-agent')
   implementation 'org.ow2.asm:asm:9.7'
+  testImplementation project(":test:framework")
+  testImplementation project(":distribution:tools:entitlement-bridge")
+  testImplementation 'org.ow2.asm:asm-util:9.7'
+}
+
+tasks.named('test').configure {
+  systemProperty "tests.security.manager", "false"
 }
 
 tasks.named('forbiddenApisMain').configure {
diff --git a/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java
new file mode 100644
index 0000000000000..d7aaa6d854e9c
--- /dev/null
+++ b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.instrumentation.impl;
+
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.util.Printer;
+import org.objectweb.asm.util.Textifier;
+import org.objectweb.asm.util.TraceClassVisitor;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+public class ASMUtils {
+    public static String bytecode2text(byte[] classBytes) {
+        ClassReader classReader = new ClassReader(classBytes);
+        StringWriter stringWriter = new StringWriter();
+        try (PrintWriter printWriter = new PrintWriter(stringWriter)) {
+            Printer printer = new Textifier(); // For a textual representation
+            TraceClassVisitor traceClassVisitor = new TraceClassVisitor(null, printer, printWriter);
+            classReader.accept(traceClassVisitor, 0);
+            return stringWriter.toString();
+        }
+    }
+}
diff --git a/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java
new file mode 100644
index 0000000000000..e807ecee4f103
--- /dev/null
+++ b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.instrumentation.impl;
+
+import org.elasticsearch.entitlement.api.EntitlementChecks;
+import org.elasticsearch.entitlement.api.EntitlementProvider;
+import org.elasticsearch.entitlement.instrumentation.InstrumentationService;
+import org.elasticsearch.entitlement.instrumentation.MethodKey;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Map;
+
+import static org.elasticsearch.entitlement.instrumentation.impl.ASMUtils.bytecode2text;
+
+/**
+ * This tests {@link InstrumenterImpl} in isolation, without a java agent.
+ * It causes the methods to be instrumented, and verifies that the instrumentation is called as expected.
+ * Problems with bytecode generation are easier to debug this way than in the context of an agent.
+ */
+@ESTestCase.WithoutSecurityManager
+public class InstrumenterTests extends ESTestCase {
+    final InstrumentationService instrumentationService = new InstrumentationServiceImpl();
+
+    private static TestEntitlementManager getTestChecks() {
+        return (TestEntitlementManager) EntitlementProvider.checks();
+    }
+
+    @Before
+    public void initialize() {
+        getTestChecks().isActive = false;
+    }
+
+    /**
+     * Contains all the virtual methods from {@link ClassToInstrument},
+     * allowing this test to call them on the dynamically loaded instrumented class.
+     */
+    public interface Testable {}
+
+    /**
+     * This is a placeholder for real class library methods.
+     * Without the java agent, we can't instrument the real methods, so we instrument this instead.
+     * <p>
+     * Methods of this class must have the same signature and the same static/virtual condition as the corresponding real method.
+     * They should assert that the arguments came through correctly.
+     * They must not throw {@link TestException}.
+     */
+    public static class ClassToInstrument implements Testable {
+        public static void systemExit(int status) {
+            assertEquals(123, status);
+        }
+    }
+
+    static final class TestException extends RuntimeException {}
+
+    /**
+     * We're not testing the permission checking logic here.
+     * This is a trivial implementation of {@link EntitlementChecks} that just always throws,
+     * just to demonstrate that the injected bytecodes succeed in calling these methods.
+     */
+    public static class TestEntitlementManager implements EntitlementChecks {
+        /**
+         * This allows us to test that the instrumentation is correct in both cases:
+         * if the check throws, and if it doesn't.
+         */
+        volatile boolean isActive;
+
+        @Override
+        public void checkSystemExit(Class<?> callerClass, int status) {
+            assertSame(InstrumenterTests.class, callerClass);
+            assertEquals(123, status);
+            throwIfActive();
+        }
+
+        private void throwIfActive() {
+            if (isActive) {
+                throw new TestException();
+            }
+        }
+    }
+
+    public void test() throws Exception {
+        // This test doesn't replace ClassToInstrument in-place but instead loads a separate
+        // class ClassToInstrument_NEW that contains the instrumentation. Because of this,
+        // we need to configure the Transformer to use a MethodKey and instrumentationMethod
+        // with slightly different signatures (using the common interface Testable) which
+        // is not what would happen when it's run by the agent.
+
+        MethodKey k1 = instrumentationService.methodKeyForTarget(ClassToInstrument.class.getMethod("systemExit", int.class));
+        Method v1 = EntitlementChecks.class.getMethod("checkSystemExit", Class.class, int.class);
+        var instrumenter = new InstrumenterImpl("_NEW", Map.of(k1, v1));
+
+        byte[] newBytecode = instrumenter.instrumentClassFile(ClassToInstrument.class).bytecodes();
+
+        if (logger.isTraceEnabled()) {
+            logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode));
+        }
+
+        Class<?> newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes(
+            ClassToInstrument.class.getName() + "_NEW",
+            newBytecode
+        );
+
+        // Before checking is active, nothing should throw
+        callStaticSystemExit(newClass, 123);
+
+        getTestChecks().isActive = true;
+
+        // After checking is activated, everything should throw
+        assertThrows(TestException.class, () -> callStaticSystemExit(newClass, 123));
+    }
+
+    /**
+     * Calling a static method of a dynamically loaded class is significantly more cumbersome
+     * than calling a virtual method.
+     */
+    private static void callStaticSystemExit(Class<?> c, int status) throws NoSuchMethodException, IllegalAccessException {
+        try {
+            c.getMethod("systemExit", int.class).invoke(null, status);
+        } catch (InvocationTargetException e) {
+            Throwable cause = e.getCause();
+            if (cause instanceof TestException n) {
+                // Sometimes we're expecting this one!
+                throw n;
+            } else {
+                throw new AssertionError(cause);
+            }
+        }
+    }
+
+    static class TestLoader extends ClassLoader {
+        TestLoader(ClassLoader parent) {
+            super(parent);
+        }
+
+        public Class<?> defineClassFromBytes(String name, byte[] bytes) {
+            return defineClass(name, bytes, 0, bytes.length);
+        }
+    }
+
+    private static final Logger logger = LogManager.getLogger(InstrumenterTests.class);
+}
diff --git a/distribution/tools/entitlement-agent/impl/src/test/resources/META-INF/services/org.elasticsearch.entitlement.api.EntitlementChecks b/distribution/tools/entitlement-agent/impl/src/test/resources/META-INF/services/org.elasticsearch.entitlement.api.EntitlementChecks
new file mode 100644
index 0000000000000..983585190b35a
--- /dev/null
+++ b/distribution/tools/entitlement-agent/impl/src/test/resources/META-INF/services/org.elasticsearch.entitlement.api.EntitlementChecks
@@ -0,0 +1,10 @@
+#
+ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ # or more contributor license agreements. Licensed under the "Elastic License
+ # 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ # Public License v 1"; you may not use this file except in compliance with, at
+ # your election, the "Elastic License 2.0", the "GNU Affero General Public
+ # License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+org.elasticsearch.entitlement.instrumentation.impl.InstrumenterTests$TestEntitlementManager
diff --git a/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java
index bb775d302c1d0..cf7991626029a 100644
--- a/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java
+++ b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java
@@ -24,6 +24,10 @@
  * to make sure it works with the entitlement granted and throws without it.
  * The only exception is {@link System#exit}, where we can't that it works without
  * terminating the JVM.
+ * <p>
+ * If you're trying to debug the instrumentation code, take a look at {@code InstrumenterTests}.
+ * That tests the bytecode portion without firing up an agent, which makes everything easier to troubleshoot.
+ * <p>
  * See {@code build.gradle} for how we set the command line arguments for this test.
  */
 @WithoutSecurityManager
diff --git a/distribution/tools/entitlement-runtime/build.gradle b/distribution/tools/entitlement-runtime/build.gradle
index 0fb7bdec883f8..55471272c1b5f 100644
--- a/distribution/tools/entitlement-runtime/build.gradle
+++ b/distribution/tools/entitlement-runtime/build.gradle
@@ -11,16 +11,12 @@ apply plugin: 'elasticsearch.publish'
 
 dependencies {
   compileOnly project(':libs:elasticsearch-core') // For @SuppressForbidden
+  compileOnly project(":libs:elasticsearch-x-content") // for parsing policy files
   compileOnly project(':server') // To access the main server module for special permission checks
   compileOnly project(':distribution:tools:entitlement-bridge')
-
   testImplementation project(":test:framework")
 }
 
 tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
-
-tasks.named('forbiddenApisMain').configure {
-  replaceSignatureFiles 'jdk-signatures'
-}
diff --git a/distribution/tools/entitlement-runtime/src/main/java/module-info.java b/distribution/tools/entitlement-runtime/src/main/java/module-info.java
index d0bfc804f8024..12e6905014512 100644
--- a/distribution/tools/entitlement-runtime/src/main/java/module-info.java
+++ b/distribution/tools/entitlement-runtime/src/main/java/module-info.java
@@ -9,6 +9,7 @@
 
 module org.elasticsearch.entitlement.runtime {
     requires org.elasticsearch.entitlement.bridge;
+    requires org.elasticsearch.xcontent;
     requires org.elasticsearch.server;
 
     exports org.elasticsearch.entitlement.runtime.api;
diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java
new file mode 100644
index 0000000000000..5b53c399cc1b7
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+/**
+ * Marker interface to ensure that only {@link Entitlement} are
+ * part of a {@link Policy}. All entitlement classes should implement
+ * this.
+ */
+public interface Entitlement {
+
+}
diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java
new file mode 100644
index 0000000000000..bb1205696b49e
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * This annotation indicates an {@link Entitlement} is available
+ * to "external" classes such as those used in plugins. Any {@link Entitlement}
+ * using this annotation is considered parseable as part of a policy file
+ * for entitlements.
+ */
+@Target(ElementType.CONSTRUCTOR)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ExternalEntitlement {
+
+    /**
+     * This is the list of parameter names that are
+     * parseable in {@link PolicyParser#parseEntitlement(String, String)}.
+     * The number and order of parameter names much match the number and order
+     * of constructor parameters as this is how the parser will pass in the
+     * parsed values from a policy file. However, the names themselves do NOT
+     * have to match the parameter names of the constructor.
+     */
+    String[] parameterNames() default {};
+}
diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java
new file mode 100644
index 0000000000000..8df199591d3e4
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Describes a file entitlement with a path and actions.
+ */
+public class FileEntitlement implements Entitlement {
+
+    public static final int READ_ACTION = 0x1;
+    public static final int WRITE_ACTION = 0x2;
+
+    private final String path;
+    private final int actions;
+
+    @ExternalEntitlement(parameterNames = { "path", "actions" })
+    public FileEntitlement(String path, List<String> actionsList) {
+        this.path = path;
+        int actionsInt = 0;
+
+        for (String actionString : actionsList) {
+            if ("read".equals(actionString)) {
+                if ((actionsInt & READ_ACTION) == READ_ACTION) {
+                    throw new IllegalArgumentException("file action [read] specified multiple times");
+                }
+                actionsInt |= READ_ACTION;
+            } else if ("write".equals(actionString)) {
+                if ((actionsInt & WRITE_ACTION) == WRITE_ACTION) {
+                    throw new IllegalArgumentException("file action [write] specified multiple times");
+                }
+                actionsInt |= WRITE_ACTION;
+            } else {
+                throw new IllegalArgumentException("unknown file action [" + actionString + "]");
+            }
+        }
+
+        this.actions = actionsInt;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        FileEntitlement that = (FileEntitlement) o;
+        return actions == that.actions && Objects.equals(path, that.path);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(path, actions);
+    }
+
+    @Override
+    public String toString() {
+        return "FileEntitlement{" + "path='" + path + '\'' + ", actions=" + actions + '}';
+    }
+}
diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java
new file mode 100644
index 0000000000000..e8bd7a3fff357
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A holder for scoped entitlements.
+ */
+public class Policy {
+
+    public final String name;
+    public final List<Scope> scopes;
+
+    public Policy(String name, List<Scope> scopes) {
+        this.name = Objects.requireNonNull(name);
+        this.scopes = Collections.unmodifiableList(Objects.requireNonNull(scopes));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Policy policy = (Policy) o;
+        return Objects.equals(name, policy.name) && Objects.equals(scopes, policy.scopes);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, scopes);
+    }
+
+    @Override
+    public String toString() {
+        return "Policy{" + "name='" + name + '\'' + ", scopes=" + scopes + '}';
+    }
+}
diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java
new file mode 100644
index 0000000000000..229ccec3b8b2c
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.yaml.YamlXContent;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.entitlement.runtime.policy.PolicyParserException.newPolicyParserException;
+
+/**
+ * A parser to parse policy files for entitlements.
+ */
+public class PolicyParser {
+
+    protected static final ParseField ENTITLEMENTS_PARSEFIELD = new ParseField("entitlements");
+
+    protected static final String entitlementPackageName = Entitlement.class.getPackage().getName();
+
+    protected final XContentParser policyParser;
+    protected final String policyName;
+
+    public PolicyParser(InputStream inputStream, String policyName) throws IOException {
+        this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream));
+        this.policyName = policyName;
+    }
+
+    public Policy parsePolicy() {
+        try {
+            if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) {
+                throw newPolicyParserException("expected object <scope name>");
+            }
+            List<Scope> scopes = new ArrayList<>();
+            while (policyParser.nextToken() != XContentParser.Token.END_OBJECT) {
+                if (policyParser.currentToken() != XContentParser.Token.FIELD_NAME) {
+                    throw newPolicyParserException("expected object <scope name>");
+                }
+                String scopeName = policyParser.currentName();
+                Scope scope = parseScope(scopeName);
+                scopes.add(scope);
+            }
+            return new Policy(policyName, scopes);
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+
+    protected Scope parseScope(String scopeName) throws IOException {
+        try {
+            if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) {
+                throw newPolicyParserException(scopeName, "expected object [" + ENTITLEMENTS_PARSEFIELD.getPreferredName() + "]");
+            }
+            if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME
+                || policyParser.currentName().equals(ENTITLEMENTS_PARSEFIELD.getPreferredName()) == false) {
+                throw newPolicyParserException(scopeName, "expected object [" + ENTITLEMENTS_PARSEFIELD.getPreferredName() + "]");
+            }
+            if (policyParser.nextToken() != XContentParser.Token.START_ARRAY) {
+                throw newPolicyParserException(scopeName, "expected array of <entitlement type>");
+            }
+            List<Entitlement> entitlements = new ArrayList<>();
+            while (policyParser.nextToken() != XContentParser.Token.END_ARRAY) {
+                if (policyParser.currentToken() != XContentParser.Token.START_OBJECT) {
+                    throw newPolicyParserException(scopeName, "expected object <entitlement type>");
+                }
+                if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME) {
+                    throw newPolicyParserException(scopeName, "expected object <entitlement type>");
+                }
+                String entitlementType = policyParser.currentName();
+                Entitlement entitlement = parseEntitlement(scopeName, entitlementType);
+                entitlements.add(entitlement);
+                if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) {
+                    throw newPolicyParserException(scopeName, "expected closing object");
+                }
+            }
+            if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) {
+                throw newPolicyParserException(scopeName, "expected closing object");
+            }
+            return new Scope(scopeName, entitlements);
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+
+    protected Entitlement parseEntitlement(String scopeName, String entitlementType) throws IOException {
+        Class<?> entitlementClass;
+        try {
+            entitlementClass = Class.forName(
+                entitlementPackageName
+                    + "."
+                    + Character.toUpperCase(entitlementType.charAt(0))
+                    + entitlementType.substring(1)
+                    + "Entitlement"
+            );
+        } catch (ClassNotFoundException cnfe) {
+            throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
+        }
+        if (Entitlement.class.isAssignableFrom(entitlementClass) == false) {
+            throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
+        }
+        Constructor<?> entitlementConstructor = entitlementClass.getConstructors()[0];
+        ExternalEntitlement entitlementMetadata = entitlementConstructor.getAnnotation(ExternalEntitlement.class);
+        if (entitlementMetadata == null) {
+            throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
+        }
+
+        if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters");
+        }
+        Map<String, Object> parsedValues = policyParser.map();
+
+        Class<?>[] parameterTypes = entitlementConstructor.getParameterTypes();
+        String[] parametersNames = entitlementMetadata.parameterNames();
+        Object[] parameterValues = new Object[parameterTypes.length];
+        for (int parameterIndex = 0; parameterIndex < parameterTypes.length; ++parameterIndex) {
+            String parameterName = parametersNames[parameterIndex];
+            Object parameterValue = parsedValues.remove(parameterName);
+            if (parameterValue == null) {
+                throw newPolicyParserException(scopeName, entitlementType, "missing entitlement parameter [" + parameterName + "]");
+            }
+            Class<?> parameterType = parameterTypes[parameterIndex];
+            if (parameterType.isAssignableFrom(parameterValue.getClass()) == false) {
+                throw newPolicyParserException(
+                    scopeName,
+                    entitlementType,
+                    "unexpected parameter type [" + parameterType.getSimpleName() + "] for entitlement parameter [" + parameterName + "]"
+                );
+            }
+            parameterValues[parameterIndex] = parameterValue;
+        }
+        if (parsedValues.isEmpty() == false) {
+            throw newPolicyParserException(scopeName, entitlementType, "extraneous entitlement parameter(s) " + parsedValues);
+        }
+
+        try {
+            return (Entitlement) entitlementConstructor.newInstance(parameterValues);
+        } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) {
+            throw new IllegalStateException("internal error");
+        }
+    }
+
+    protected PolicyParserException newPolicyParserException(String message) {
+        return PolicyParserException.newPolicyParserException(policyParser.getTokenLocation(), policyName, message);
+    }
+
+    protected PolicyParserException newPolicyParserException(String scopeName, String message) {
+        return PolicyParserException.newPolicyParserException(policyParser.getTokenLocation(), policyName, scopeName, message);
+    }
+
+    protected PolicyParserException newPolicyParserException(String scopeName, String entitlementType, String message) {
+        return PolicyParserException.newPolicyParserException(
+            policyParser.getTokenLocation(),
+            policyName,
+            scopeName,
+            entitlementType,
+            message
+        );
+    }
+}
diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java
new file mode 100644
index 0000000000000..5dfa12f11d0be
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import org.elasticsearch.xcontent.XContentLocation;
+
+/**
+ * An exception specifically for policy parsing errors.
+ */
+public class PolicyParserException extends RuntimeException {
+
+    public static PolicyParserException newPolicyParserException(XContentLocation location, String policyName, String message) {
+        return new PolicyParserException(
+            "[" + location.lineNumber() + ":" + location.columnNumber() + "] policy parsing error for [" + policyName + "]: " + message
+        );
+    }
+
+    public static PolicyParserException newPolicyParserException(
+        XContentLocation location,
+        String policyName,
+        String scopeName,
+        String message
+    ) {
+        if (scopeName == null) {
+            return new PolicyParserException(
+                "[" + location.lineNumber() + ":" + location.columnNumber() + "] policy parsing error for [" + policyName + "]: " + message
+            );
+        } else {
+            return new PolicyParserException(
+                "["
+                    + location.lineNumber()
+                    + ":"
+                    + location.columnNumber()
+                    + "] policy parsing error for ["
+                    + policyName
+                    + "] in scope ["
+                    + scopeName
+                    + "]: "
+                    + message
+            );
+        }
+    }
+
+    public static PolicyParserException newPolicyParserException(
+        XContentLocation location,
+        String policyName,
+        String scopeName,
+        String entitlementType,
+        String message
+    ) {
+        if (scopeName == null) {
+            return new PolicyParserException(
+                "["
+                    + location.lineNumber()
+                    + ":"
+                    + location.columnNumber()
+                    + "] policy parsing error for ["
+                    + policyName
+                    + "] for entitlement type ["
+                    + entitlementType
+                    + "]: "
+                    + message
+            );
+        } else {
+            return new PolicyParserException(
+                "["
+                    + location.lineNumber()
+                    + ":"
+                    + location.columnNumber()
+                    + "] policy parsing error for ["
+                    + policyName
+                    + "] in scope ["
+                    + scopeName
+                    + "] for entitlement type ["
+                    + entitlementType
+                    + "]: "
+                    + message
+            );
+        }
+    }
+
+    private PolicyParserException(String message) {
+        super(message);
+    }
+}
diff --git a/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java
new file mode 100644
index 0000000000000..0fe63eb8da1b7
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A holder for entitlements within a single scope.
+ */
+public class Scope {
+
+    public final String name;
+    public final List<Entitlement> entitlements;
+
+    public Scope(String name, List<Entitlement> entitlements) {
+        this.name = Objects.requireNonNull(name);
+        this.entitlements = Collections.unmodifiableList(Objects.requireNonNull(entitlements));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Scope scope = (Scope) o;
+        return Objects.equals(name, scope.name) && Objects.equals(entitlements, scope.entitlements);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, entitlements);
+    }
+
+    @Override
+    public String toString() {
+        return "Scope{" + "name='" + name + '\'' + ", entitlements=" + entitlements + '}';
+    }
+}
diff --git a/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java
new file mode 100644
index 0000000000000..b21d206f3eb6a
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class PolicyParserFailureTests extends ESTestCase {
+
+    public void testParserSyntaxFailures() {
+        PolicyParserException ppe = expectThrows(
+            PolicyParserException.class,
+            () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml")
+                .parsePolicy()
+        );
+        assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object <scope name>", ppe.getMessage());
+    }
+
+    public void testEntitlementDoesNotExist() throws IOException {
+        PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              entitlements:
+                - does_not_exist: {}
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+        assertEquals(
+            "[3:7] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: "
+                + "unknown entitlement type [does_not_exist]",
+            ppe.getMessage()
+        );
+    }
+
+    public void testEntitlementMissingParameter() throws IOException {
+        PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              entitlements:
+                - file: {}
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+        assertEquals(
+            "[3:14] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+                + "for entitlement type [file]: missing entitlement parameter [path]",
+            ppe.getMessage()
+        );
+
+        ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              entitlements:
+                - file:
+                    path: test-path
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+        assertEquals(
+            "[5:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+                + "for entitlement type [file]: missing entitlement parameter [actions]",
+            ppe.getMessage()
+        );
+    }
+
+    public void testEntitlementExtraneousParameter() throws IOException {
+        PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              entitlements:
+                - file:
+                    path: test-path
+                    actions:
+                      - read
+                    extra: test
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+        assertEquals(
+            "[8:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+                + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}",
+            ppe.getMessage()
+        );
+    }
+}
diff --git a/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
new file mode 100644
index 0000000000000..40016b2e3027e
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+public class PolicyParserTests extends ESTestCase {
+
+    public void testPolicyBuilder() throws IOException {
+        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml")
+            .parsePolicy();
+        Policy builtPolicy = new Policy(
+            "test-policy.yaml",
+            List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write")))))
+        );
+        assertEquals(parsedPolicy, builtPolicy);
+    }
+}
diff --git a/distribution/tools/entitlement-runtime/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml b/distribution/tools/entitlement-runtime/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml
new file mode 100644
index 0000000000000..b58287cfc83b7
--- /dev/null
+++ b/distribution/tools/entitlement-runtime/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml
@@ -0,0 +1,7 @@
+entitlement-module-name:
+  entitlements:
+    - file:
+        path: "test/path/to/file"
+        actions:
+          - "read"
+          - "write"
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index b65b974cd6b69..bdb0704fcd880 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.12.0
-:lucene_version_path:   9_12_0
+:lucene_version:        10.0.0
+:lucene_version_path:   10_0_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar
diff --git a/docs/changelog/111684.yaml b/docs/changelog/111684.yaml
deleted file mode 100644
index 32edb5723cb0a..0000000000000
--- a/docs/changelog/111684.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 111684
-summary: Write downloaded model parts async
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/112250.yaml b/docs/changelog/112250.yaml
new file mode 100644
index 0000000000000..edbb5667d4b9d
--- /dev/null
+++ b/docs/changelog/112250.yaml
@@ -0,0 +1,5 @@
+pr: 112250
+summary: Do not exclude empty arrays or empty objects in source filtering
+area: Search
+type: bug
+issues: [109668]
diff --git a/docs/changelog/112761.yaml b/docs/changelog/112761.yaml
deleted file mode 100644
index fe63f38f365a4..0000000000000
--- a/docs/changelog/112761.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 112761
-summary: Fix collapse interaction with stored fields
-area: Search
-type: bug
-issues:
- - 112646
diff --git a/docs/changelog/112881.yaml b/docs/changelog/112881.yaml
new file mode 100644
index 0000000000000..a8a0d542f8201
--- /dev/null
+++ b/docs/changelog/112881.yaml
@@ -0,0 +1,5 @@
+pr: 112881
+summary: "ESQL: Remove parent from `FieldAttribute`"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/113123.yaml b/docs/changelog/113123.yaml
deleted file mode 100644
index 43008eaa80f43..0000000000000
--- a/docs/changelog/113123.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 113123
-summary: "ES|QL: Skip CASE function from `InferIsNotNull` rule checks"
-area: ES|QL
-type: bug
-issues:
- - 112704
diff --git a/docs/changelog/113129.yaml b/docs/changelog/113129.yaml
deleted file mode 100644
index d88d86387ac10..0000000000000
--- a/docs/changelog/113129.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 113129
-summary: Fix `needsScore` computation in `GlobalOrdCardinalityAggregator`
-area: Aggregations
-type: bug
-issues:
- - 112975
diff --git a/docs/changelog/113237.yaml b/docs/changelog/113237.yaml
new file mode 100644
index 0000000000000..45343dbf17114
--- /dev/null
+++ b/docs/changelog/113237.yaml
@@ -0,0 +1,5 @@
+pr: 113237
+summary: Retry throttled snapshot deletions
+area: Snapshot/Restore
+type: bug
+issues: []
diff --git a/docs/changelog/113266.yaml b/docs/changelog/113266.yaml
deleted file mode 100644
index d423387d45738..0000000000000
--- a/docs/changelog/113266.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 113266
-summary: "[M] Fix error message formatting"
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/113374.yaml b/docs/changelog/113374.yaml
new file mode 100644
index 0000000000000..f1d5750de0f60
--- /dev/null
+++ b/docs/changelog/113374.yaml
@@ -0,0 +1,5 @@
+pr: 113374
+summary: Add ESQL match function
+area: ES|QL
+type: feature
+issues: []
diff --git a/docs/changelog/113437.yaml b/docs/changelog/113437.yaml
deleted file mode 100644
index 98831958e63f8..0000000000000
--- a/docs/changelog/113437.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 113437
-summary: Fix check on E5 model platform compatibility
-area: Machine Learning
-type: bug
-issues:
- - 113577
diff --git a/docs/changelog/113482.yaml b/docs/changelog/113482.yaml
new file mode 100644
index 0000000000000..cb5823f0ccfcc
--- /dev/null
+++ b/docs/changelog/113482.yaml
@@ -0,0 +1,27 @@
+pr: 113482
+summary: The 'persian' analyzer has stemmer by default
+area: Analysis
+type: breaking
+issues:
+- 113050
+breaking:
+  title: The 'persian' analyzer has stemmer by default
+  area: Analysis
+  details: >-
+    Lucene 10 has added a final stemming step to its PersianAnalyzer that Elasticsearch
+    exposes as 'persian' analyzer. Existing indices will keep the old
+    non-stemming behaviour while new indices will see the updated behaviour with
+    added stemming.
+    Users that wish to maintain the non-stemming behaviour need to define their
+    own analyzer as outlined in
+    https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer.
+    Users that wish to use the new stemming behaviour for existing indices will
+    have to reindex their data.
+  impact: >-
+    Indexing with the 'persian' analyzer will produce slightly different tokens.
+    Users should check if this impacts their search results. If they wish to
+    maintain the legacy non-stemming behaviour they can define their own
+    analyzer equivalent as explained in
+    https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer.
+  notable: false
+
diff --git a/docs/changelog/113614.yaml b/docs/changelog/113614.yaml
new file mode 100644
index 0000000000000..bd9dcb3e38772
--- /dev/null
+++ b/docs/changelog/113614.yaml
@@ -0,0 +1,18 @@
+pr: 113614
+summary: The 'german2' stemmer is now an alias for the 'german' snowball stemmer
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: The "german2" snowball stemmer is now an alias for the "german" stemmer
+  area: Analysis
+  details: >-
+    Lucene 10 has merged the improved "german2" snowball language stemmer with the
+    "german" stemmer. For Elasticsearch, "german2" is now a deprecated alias for
+    "german". This may results in slightly different tokens being generated for
+    terms with umlaut substitution (like "ue" for "ü" etc...)
+  impact: >-
+    Replace usages of "german2" with "german" in analysis configuration. Old
+    indices that use the "german" stemmer should be reindexed if possible.
+  notable: false
+
diff --git a/docs/changelog/113697.yaml b/docs/changelog/113697.yaml
deleted file mode 100644
index 1362e01fcc89b..0000000000000
--- a/docs/changelog/113697.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 113697
-summary: Handle parsing ingest processors where definition is not a object
-area: Machine Learning
-type: bug
-issues:
- - 113615
diff --git a/docs/changelog/113699.yaml b/docs/changelog/113699.yaml
deleted file mode 100644
index 3876c8147e7eb..0000000000000
--- a/docs/changelog/113699.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 113699
-summary: "[ESQL] Fix init value in max float aggregation"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/113735.yaml b/docs/changelog/113735.yaml
new file mode 100644
index 0000000000000..4f6579c7cb9e0
--- /dev/null
+++ b/docs/changelog/113735.yaml
@@ -0,0 +1,28 @@
+pr: 113735
+summary: "ESQL: Introduce per agg filter"
+area: ES|QL
+type: feature
+issues: []
+highlight:
+  title: "ESQL: Introduce per agg filter"
+  body: |-
+    Add support for aggregation scoped filters that work dynamically on the
+    data in each group.
+    
+    [source,esql]
+    ----
+    | STATS success = COUNT(*) WHERE 200 <= code AND code < 300,
+            redirect = COUNT(*) WHERE 300 <= code AND code < 400,
+            client_err = COUNT(*) WHERE 400 <= code AND code < 500,
+            server_err = COUNT(*) WHERE 500 <= code AND code < 600,
+            total_count = COUNT(*)
+    ----
+    
+    Implementation wise, the base AggregateFunction has been extended to 
+    allow a filter to be passed on. This is required to incorporate the
+    filter as part of the aggregate equality/identity which would fail with 
+    the filter as an external component.
+    As part of the process, the serialization for the existing aggregations
+    had to be fixed so AggregateFunction implementations so that it
+    delegates to their parent first.
+  notable: true
diff --git a/docs/changelog/113846.yaml b/docs/changelog/113846.yaml
deleted file mode 100644
index 5fdd56e98d706..0000000000000
--- a/docs/changelog/113846.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 113846
-summary: Don't validate internal stats if they are empty
-area: Aggregations
-type: bug
-issues:
- - 113811
diff --git a/docs/changelog/113869.yaml b/docs/changelog/113869.yaml
deleted file mode 100644
index f1cd1ec423966..0000000000000
--- a/docs/changelog/113869.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 113869
-summary: Upgrade protobufer to 3.25.5
-area: Snapshot/Restore
-type: upgrade
-issues: []
diff --git a/docs/changelog/113920.yaml b/docs/changelog/113920.yaml
new file mode 100644
index 0000000000000..4699ae6d7dd65
--- /dev/null
+++ b/docs/changelog/113920.yaml
@@ -0,0 +1,5 @@
+pr: 113920
+summary: Add initial support for `semantic_text` field type
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/113961.yaml b/docs/changelog/113961.yaml
deleted file mode 100644
index 24cb1f45f029e..0000000000000
--- a/docs/changelog/113961.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 113961
-summary: "[ESQL] Support datetime data type in Least and Greatest functions"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/113975.yaml b/docs/changelog/113975.yaml
new file mode 100644
index 0000000000000..632ba038271bb
--- /dev/null
+++ b/docs/changelog/113975.yaml
@@ -0,0 +1,19 @@
+pr: 113975
+summary: JDK locale database change
+area: Mapping
+type: breaking
+issues: []
+breaking:
+  title: JDK locale database change
+  area: Mapping
+  details: |
+    {es} 8.16 changes the version of the JDK that is included from version 22 to version 23. This changes the locale database that is used by Elasticsearch from the COMPAT database to the CLDR database. This change can cause significant differences to the textual date formats accepted by Elasticsearch, and to calculated week-dates.
+
+    If you run {es} 8.16 on JDK version 22 or below, it will use the COMPAT locale database to match the behavior of 8.15. However, starting with {es} 9.0, {es} will use the CLDR database regardless of JDK version it is run on.
+  impact: |
+    This affects you if you use custom date formats using textual or week-date field specifiers. If you use date fields or calculated week-dates that change between the COMPAT and CLDR databases, then this change will cause Elasticsearch to reject previously valid date fields as invalid data. You might need to modify your ingest or output integration code to account for the differences between these two JDK versions.
+
+    Starting in version 8.15.2, Elasticsearch will log deprecation warnings if you are using date format specifiers that might change on upgrading to JDK 23. These warnings are visible in Kibana.
+
+    For detailed guidance, refer to <<custom-date-format-locales,Differences in locale information between JDK versions>> and the https://ela.st/jdk-23-locales[Elastic blog].
+  notable: true
diff --git a/docs/changelog/114021.yaml b/docs/changelog/114021.yaml
new file mode 100644
index 0000000000000..e9dab5dce5685
--- /dev/null
+++ b/docs/changelog/114021.yaml
@@ -0,0 +1,5 @@
+pr: 114021
+summary: "ESQL: Speed up grouping by bytes"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114116.yaml b/docs/changelog/114116.yaml
deleted file mode 100644
index 8d1c9e162ae23..0000000000000
--- a/docs/changelog/114116.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 114116
-summary: "ES|QL: Ensure minimum capacity for `PlanStreamInput` caches"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/114124.yaml b/docs/changelog/114124.yaml
new file mode 100644
index 0000000000000..c812c6a468902
--- /dev/null
+++ b/docs/changelog/114124.yaml
@@ -0,0 +1,18 @@
+pr: 114124
+summary: The Korean dictionary for Nori has been updated 
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: The Korean dictionary for Nori has been updated
+  area: Analysis
+  details: >-
+    Lucene 10 ships with an updated Korean dictionary (mecab-ko-dic-2.1.1). 
+    For details see https://github.com/apache/lucene/issues/11452. Users
+    experiencing changes in search behaviour on existing data are advised to
+    reindex.
+  impact: >-
+    The change is small and should generally provide better analysis results.
+    Existing indices for full-text use cases should be reindexed though.
+  notable: false
+
diff --git a/docs/changelog/114146.yaml b/docs/changelog/114146.yaml
new file mode 100644
index 0000000000000..be2096a64105c
--- /dev/null
+++ b/docs/changelog/114146.yaml
@@ -0,0 +1,20 @@
+pr: 114146 
+summary: Snowball stemmers have been upgraded 
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: Snowball stemmers have been upgraded
+  area: Analysis
+  details: >-
+    Lucene 10 ships with an upgrade of its Snowball stemmers. 
+    For details see https://github.com/apache/lucene/issues/13209. Users using
+    Snowball stemmers that are experiencing changes in search behaviour on 
+    existing data are advised to reindex.
+  impact: >-
+    The upgrade should generally provide improved stemming results. Small changes
+    in token analysis can lead to mismatches with previously index data, so
+    existing indices using Snowball stemmers as part of their analysis chain
+    should be reindexed.
+  notable: false
+
diff --git a/docs/changelog/114168.yaml b/docs/changelog/114168.yaml
new file mode 100644
index 0000000000000..58f1ab7110e7d
--- /dev/null
+++ b/docs/changelog/114168.yaml
@@ -0,0 +1,5 @@
+pr: 114168
+summary: Add a query rules tester API call
+area: Relevance
+type: enhancement
+issues: []
diff --git a/docs/changelog/114264.yaml b/docs/changelog/114264.yaml
deleted file mode 100644
index fe421f6422830..0000000000000
--- a/docs/changelog/114264.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 114264
-summary: "Fix analyzed wildcard query in simple_query_string when disjunctions is empty"
-area: Search
-type: bug
-issues: [114185]
diff --git a/docs/changelog/114271.yaml b/docs/changelog/114271.yaml
new file mode 100644
index 0000000000000..7b47b922ff811
--- /dev/null
+++ b/docs/changelog/114271.yaml
@@ -0,0 +1,5 @@
+pr: 114271
+summary: "[ES|QL] Skip validating remote cluster index names in parser"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/114295.yaml b/docs/changelog/114295.yaml
new file mode 100644
index 0000000000000..2acdc293a206c
--- /dev/null
+++ b/docs/changelog/114295.yaml
@@ -0,0 +1,5 @@
+pr: 114295
+summary: "Reprocess operator file settings when settings service starts, due to node restart or master node change"
+area: Infra/Settings
+type: enhancement
+issues: [ ]
diff --git a/docs/changelog/114337.yaml b/docs/changelog/114337.yaml
deleted file mode 100644
index ec55be8bb179b..0000000000000
--- a/docs/changelog/114337.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 114337
-summary: "Enables cluster state role mapper, to include ECK operator-defined role mappings in role resolution"
-area: Authentication
-type: bug
-issues: []
diff --git a/docs/changelog/114382.yaml b/docs/changelog/114382.yaml
new file mode 100644
index 0000000000000..9f572e14f4737
--- /dev/null
+++ b/docs/changelog/114382.yaml
@@ -0,0 +1,5 @@
+pr: 114382
+summary: "[ES|QL] Add hypot function"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114407.yaml b/docs/changelog/114407.yaml
new file mode 100644
index 0000000000000..4c1134a9d3834
--- /dev/null
+++ b/docs/changelog/114407.yaml
@@ -0,0 +1,6 @@
+pr: 114407
+summary: Fix synthetic source handling for `bit` type in `dense_vector` field
+area: Search
+type: bug
+issues:
+  - 114402
diff --git a/docs/changelog/114439.yaml b/docs/changelog/114439.yaml
new file mode 100644
index 0000000000000..fd097d02f885f
--- /dev/null
+++ b/docs/changelog/114439.yaml
@@ -0,0 +1,5 @@
+pr: 114439
+summary: Adding new bbq index types behind a feature flag
+area: Vector Search
+type: feature
+issues: []
diff --git a/docs/changelog/114453.yaml b/docs/changelog/114453.yaml
new file mode 100644
index 0000000000000..0d5345ad9d2a6
--- /dev/null
+++ b/docs/changelog/114453.yaml
@@ -0,0 +1,5 @@
+pr: 114453
+summary: Switch default chunking strategy to sentence
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114457.yaml b/docs/changelog/114457.yaml
new file mode 100644
index 0000000000000..9558c41852f69
--- /dev/null
+++ b/docs/changelog/114457.yaml
@@ -0,0 +1,6 @@
+pr: 114457
+summary: "[Inference API] Introduce Update API to change some aspects of existing\
+  \ inference endpoints"
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114482.yaml b/docs/changelog/114482.yaml
new file mode 100644
index 0000000000000..a5e2e981f7adc
--- /dev/null
+++ b/docs/changelog/114482.yaml
@@ -0,0 +1,5 @@
+pr: 114482
+summary: Remove snapshot build restriction for match and qstr functions
+area: ES|QL
+type: feature
+issues: []
diff --git a/docs/changelog/114549.yaml b/docs/changelog/114549.yaml
new file mode 100644
index 0000000000000..a6bdbba93876b
--- /dev/null
+++ b/docs/changelog/114549.yaml
@@ -0,0 +1,5 @@
+pr: 114549
+summary: Send mid-stream errors to users
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/114566.yaml b/docs/changelog/114566.yaml
new file mode 100644
index 0000000000000..6007152bb26ca
--- /dev/null
+++ b/docs/changelog/114566.yaml
@@ -0,0 +1,5 @@
+pr: 114566
+summary: Use Azure blob batch API to delete blobs in batches
+area: Distributed
+type: enhancement
+issues: []
diff --git a/docs/changelog/114596.yaml b/docs/changelog/114596.yaml
new file mode 100644
index 0000000000000..a36978dcacd8c
--- /dev/null
+++ b/docs/changelog/114596.yaml
@@ -0,0 +1,5 @@
+pr: 114596
+summary: Stream Google Completion
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114601.yaml b/docs/changelog/114601.yaml
new file mode 100644
index 0000000000000..d2f563d62a639
--- /dev/null
+++ b/docs/changelog/114601.yaml
@@ -0,0 +1,6 @@
+pr: 114601
+summary: Support semantic_text in object fields
+area: Vector Search
+type: bug
+issues:
+ - 114401
diff --git a/docs/changelog/114620.yaml b/docs/changelog/114620.yaml
new file mode 100644
index 0000000000000..92498db92061f
--- /dev/null
+++ b/docs/changelog/114620.yaml
@@ -0,0 +1,5 @@
+pr: 114620
+summary: "ES|QL: add metrics for functions"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114623.yaml b/docs/changelog/114623.yaml
new file mode 100644
index 0000000000000..817a8e874bcc0
--- /dev/null
+++ b/docs/changelog/114623.yaml
@@ -0,0 +1,5 @@
+pr: 114623
+summary: Preserve thread context when waiting for segment generation in RTG
+area: CRUD
+type: bug
+issues: []
diff --git a/docs/changelog/114638.yaml b/docs/changelog/114638.yaml
new file mode 100644
index 0000000000000..0386aacfe3e18
--- /dev/null
+++ b/docs/changelog/114638.yaml
@@ -0,0 +1,7 @@
+pr: 114638
+summary: "ES|QL: Restrict sorting for `_source` and counter field types"
+area: ES|QL
+type: bug
+issues:
+ - 114423
+ - 111976
diff --git a/docs/changelog/114665.yaml b/docs/changelog/114665.yaml
new file mode 100644
index 0000000000000..b90bb799bd896
--- /dev/null
+++ b/docs/changelog/114665.yaml
@@ -0,0 +1,6 @@
+pr: 114665
+summary: Fixing remote ENRICH by pushing the Enrich inside `FragmentExec`
+area: ES|QL
+type: bug
+issues:
+  - 105095
diff --git a/docs/changelog/114683.yaml b/docs/changelog/114683.yaml
new file mode 100644
index 0000000000000..a677e65a12b0e
--- /dev/null
+++ b/docs/changelog/114683.yaml
@@ -0,0 +1,5 @@
+pr: 114683
+summary: Default inference endpoint for the multilingual-e5-small model
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114715.yaml b/docs/changelog/114715.yaml
new file mode 100644
index 0000000000000..0894cb2fa42ca
--- /dev/null
+++ b/docs/changelog/114715.yaml
@@ -0,0 +1,5 @@
+pr: 114715
+summary: Ignore unrecognized openai sse fields
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/114719.yaml b/docs/changelog/114719.yaml
new file mode 100644
index 0000000000000..477d656d5b979
--- /dev/null
+++ b/docs/changelog/114719.yaml
@@ -0,0 +1,5 @@
+pr: 114719
+summary: Wait for allocation on scale up
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114732.yaml b/docs/changelog/114732.yaml
new file mode 100644
index 0000000000000..42176cdbda443
--- /dev/null
+++ b/docs/changelog/114732.yaml
@@ -0,0 +1,5 @@
+pr: 114732
+summary: Stream Bedrock Completion
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114741.yaml b/docs/changelog/114741.yaml
new file mode 100644
index 0000000000000..ae45c183cddf9
--- /dev/null
+++ b/docs/changelog/114741.yaml
@@ -0,0 +1,5 @@
+pr: 114741
+summary: Upgrade to Lucene 10
+area: Search
+type: upgrade
+issues: []
diff --git a/docs/changelog/114742.yaml b/docs/changelog/114742.yaml
new file mode 100644
index 0000000000000..5bd3dad4400b8
--- /dev/null
+++ b/docs/changelog/114742.yaml
@@ -0,0 +1,5 @@
+pr: 114742
+summary: Adding support for additional mapping to simulate ingest API
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/114750.yaml b/docs/changelog/114750.yaml
new file mode 100644
index 0000000000000..f7a3c8c283934
--- /dev/null
+++ b/docs/changelog/114750.yaml
@@ -0,0 +1,5 @@
+pr: 114750
+summary: Create an ml node inference endpoint referencing an existing model
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114774.yaml b/docs/changelog/114774.yaml
new file mode 100644
index 0000000000000..1becfe427fda0
--- /dev/null
+++ b/docs/changelog/114774.yaml
@@ -0,0 +1,5 @@
+pr: 114774
+summary: "ESQL: Add support for multivalue fields in Arrow output"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114784.yaml b/docs/changelog/114784.yaml
new file mode 100644
index 0000000000000..24ebe8b5fc09a
--- /dev/null
+++ b/docs/changelog/114784.yaml
@@ -0,0 +1,5 @@
+pr: 114784
+summary: "[ES|QL] make named parameter for identifier and pattern snapshot"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/114813.yaml b/docs/changelog/114813.yaml
new file mode 100644
index 0000000000000..1595b004178c4
--- /dev/null
+++ b/docs/changelog/114813.yaml
@@ -0,0 +1,5 @@
+pr: 114813
+summary: Retry `S3BlobContainer#getRegister` on all exceptions
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/114819.yaml b/docs/changelog/114819.yaml
new file mode 100644
index 0000000000000..f8d03f7024801
--- /dev/null
+++ b/docs/changelog/114819.yaml
@@ -0,0 +1,6 @@
+pr: 114819
+summary: Don't use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash`
+area: EQL
+type: bug
+issues:
+ - 114599
diff --git a/docs/changelog/114836.yaml b/docs/changelog/114836.yaml
new file mode 100644
index 0000000000000..6f21d3bfb9327
--- /dev/null
+++ b/docs/changelog/114836.yaml
@@ -0,0 +1,6 @@
+pr: 114836
+summary: Support multi-valued fields in compute engine for ST_DISTANCE
+area: ES|QL
+type: enhancement
+issues:
+ - 112910
diff --git a/docs/changelog/114837.yaml b/docs/changelog/114837.yaml
new file mode 100644
index 0000000000000..313d88f92282c
--- /dev/null
+++ b/docs/changelog/114837.yaml
@@ -0,0 +1,5 @@
+pr: 114837
+summary: Add warning headers for ingest pipelines containing special characters
+area: Ingest Node
+type: bug
+issues: [ 104411 ]
diff --git a/docs/changelog/114848.yaml b/docs/changelog/114848.yaml
new file mode 100644
index 0000000000000..db41e8496f787
--- /dev/null
+++ b/docs/changelog/114848.yaml
@@ -0,0 +1,5 @@
+pr: 114848
+summary: "ESQL: Fix grammar changes around per agg filtering"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/114854.yaml b/docs/changelog/114854.yaml
new file mode 100644
index 0000000000000..144a10ba85043
--- /dev/null
+++ b/docs/changelog/114854.yaml
@@ -0,0 +1,10 @@
+pr: 114854
+summary: Adding deprecation warnings for rrf using rank and `sub_searches`
+area: Search
+type: deprecation
+issues: []
+deprecation:
+  title: Adding deprecation warnings for rrf using rank and `sub_searches`
+  area: REST API
+  details: Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter
+  impact: Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter.
diff --git a/docs/changelog/114856.yaml b/docs/changelog/114856.yaml
new file mode 100644
index 0000000000000..da7fae3ee18ea
--- /dev/null
+++ b/docs/changelog/114856.yaml
@@ -0,0 +1,5 @@
+pr: 114856
+summary: "OTel mappings: avoid metrics to be rejected when attributes are malformed"
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/114869.yaml b/docs/changelog/114869.yaml
new file mode 100644
index 0000000000000..755418e7ce4d9
--- /dev/null
+++ b/docs/changelog/114869.yaml
@@ -0,0 +1,5 @@
+pr: 114869
+summary: Standardize error code when bulk body is invalid
+area: CRUD
+type: bug
+issues: []
diff --git a/docs/changelog/114888.yaml b/docs/changelog/114888.yaml
new file mode 100644
index 0000000000000..6b99eb82d10f3
--- /dev/null
+++ b/docs/changelog/114888.yaml
@@ -0,0 +1,6 @@
+pr: 114888
+summary: Fix ST_CENTROID_AGG when no records are aggregated
+area: ES|QL
+type: bug
+issues:
+ - 106025
diff --git a/docs/changelog/114899.yaml b/docs/changelog/114899.yaml
new file mode 100644
index 0000000000000..399aa5cf35409
--- /dev/null
+++ b/docs/changelog/114899.yaml
@@ -0,0 +1,5 @@
+pr: 114899
+summary: "ES|QL: Fix stats by constant expression"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/114924.yaml b/docs/changelog/114924.yaml
new file mode 100644
index 0000000000000..536f446ef790d
--- /dev/null
+++ b/docs/changelog/114924.yaml
@@ -0,0 +1,5 @@
+pr: 114924
+summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader`
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/114951.yaml b/docs/changelog/114951.yaml
new file mode 100644
index 0000000000000..4d40a063e2b02
--- /dev/null
+++ b/docs/changelog/114951.yaml
@@ -0,0 +1,5 @@
+pr: 114951
+summary: Expose cluster-state role mappings in APIs
+area: Authentication
+type: bug
+issues: []
diff --git a/docs/changelog/114990.yaml b/docs/changelog/114990.yaml
new file mode 100644
index 0000000000000..2575942d15bf5
--- /dev/null
+++ b/docs/changelog/114990.yaml
@@ -0,0 +1,6 @@
+pr: 114990
+summary: Allow for querries on `_tier` to skip shards in the `can_match` phase
+area: Search
+type: bug
+issues:
+ - 114910
diff --git a/docs/changelog/115031.yaml b/docs/changelog/115031.yaml
new file mode 100644
index 0000000000000..d8d6e1a3f8166
--- /dev/null
+++ b/docs/changelog/115031.yaml
@@ -0,0 +1,5 @@
+pr: 115031
+summary: Bool query early termination should also consider `must_not` clauses
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/115041.yaml b/docs/changelog/115041.yaml
new file mode 100644
index 0000000000000..f4c047c1569ec
--- /dev/null
+++ b/docs/changelog/115041.yaml
@@ -0,0 +1,6 @@
+pr: 115041
+summary: Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity`
+  to 100_000
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/115048.yaml b/docs/changelog/115048.yaml
new file mode 100644
index 0000000000000..10844b83c6d01
--- /dev/null
+++ b/docs/changelog/115048.yaml
@@ -0,0 +1,5 @@
+pr: 115048
+summary: Add timeout and cancellation check to rescore phase
+area: Ranking
+type: enhancement
+issues: []
diff --git a/docs/changelog/115061.yaml b/docs/changelog/115061.yaml
new file mode 100644
index 0000000000000..7d40d5ae2629e
--- /dev/null
+++ b/docs/changelog/115061.yaml
@@ -0,0 +1,5 @@
+pr: 115061
+summary: "[ES|QL] Simplify syntax of named parameter for identifier and pattern"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/115102.yaml b/docs/changelog/115102.yaml
new file mode 100644
index 0000000000000..f679bb6c223a6
--- /dev/null
+++ b/docs/changelog/115102.yaml
@@ -0,0 +1,6 @@
+pr: 115102
+summary: Watch Next Run Interval Resets On Shard Move or Node Restart
+area: Watcher
+type: bug
+issues:
+ - 111433
diff --git a/docs/changelog/115117.yaml b/docs/changelog/115117.yaml
new file mode 100644
index 0000000000000..de2defcd46afd
--- /dev/null
+++ b/docs/changelog/115117.yaml
@@ -0,0 +1,6 @@
+pr: 115117
+summary: Report JVM stats for all memory pools (97046)
+area: Infra/Core
+type: bug
+issues:
+ - 97046
diff --git a/docs/changelog/115147.yaml b/docs/changelog/115147.yaml
new file mode 100644
index 0000000000000..36f40bba1da17
--- /dev/null
+++ b/docs/changelog/115147.yaml
@@ -0,0 +1,5 @@
+pr: 115147
+summary: Fix IPinfo geolocation schema
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/115181.yaml b/docs/changelog/115181.yaml
new file mode 100644
index 0000000000000..65f59d5ed0add
--- /dev/null
+++ b/docs/changelog/115181.yaml
@@ -0,0 +1,5 @@
+pr: 115181
+summary: Always check the parent breaker with zero bytes in `PreallocatedCircuitBreakerService`
+area: Aggregations
+type: bug
+issues: []
diff --git a/docs/changelog/115194.yaml b/docs/changelog/115194.yaml
new file mode 100644
index 0000000000000..0b201b9f89aa5
--- /dev/null
+++ b/docs/changelog/115194.yaml
@@ -0,0 +1,7 @@
+pr: 115194
+summary: Update APM Java Agent to support JDK 23
+area: Infra/Metrics
+type: upgrade
+issues:
+ - 115101
+ - 115100
diff --git a/docs/changelog/115241.yaml b/docs/changelog/115241.yaml
new file mode 100644
index 0000000000000..b7119d7f6aaeb
--- /dev/null
+++ b/docs/changelog/115241.yaml
@@ -0,0 +1,6 @@
+pr: 115241
+summary: "[Security Solution] Add `create_index` to `kibana_system` role for index/DS\
+  \ `.logs-endpoint.action.responses-*`"
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/115245.yaml b/docs/changelog/115245.yaml
new file mode 100644
index 0000000000000..294328567c3aa
--- /dev/null
+++ b/docs/changelog/115245.yaml
@@ -0,0 +1,8 @@
+pr: 115245
+summary: "ESQL: Fix `REVERSE` with backspace character"
+area: ES|QL
+type: bug
+issues:
+ - 114372
+ - 115227
+ - 115228
diff --git a/docs/changelog/115308.yaml b/docs/changelog/115308.yaml
new file mode 100644
index 0000000000000..163f0232a3e58
--- /dev/null
+++ b/docs/changelog/115308.yaml
@@ -0,0 +1,6 @@
+pr: 115308
+summary: "ESQL: Disable pushdown of WHERE past STATS"
+area: ES|QL
+type: bug
+issues:
+ - 115281
diff --git a/docs/changelog/115312.yaml b/docs/changelog/115312.yaml
new file mode 100644
index 0000000000000..acf6bbc69c36c
--- /dev/null
+++ b/docs/changelog/115312.yaml
@@ -0,0 +1,6 @@
+pr: 115312
+summary: "ESQL: Fix filtered grouping on ords"
+area: ES|QL
+type: bug
+issues:
+ - 114897
diff --git a/docs/changelog/115317.yaml b/docs/changelog/115317.yaml
new file mode 100644
index 0000000000000..153f7a52f0674
--- /dev/null
+++ b/docs/changelog/115317.yaml
@@ -0,0 +1,5 @@
+pr: 115317
+summary: Revert "Add `ResolvedExpression` wrapper"
+area: Indices APIs
+type: bug
+issues: []
diff --git a/docs/changelog/115359.yaml b/docs/changelog/115359.yaml
new file mode 100644
index 0000000000000..65b3086dfc8d0
--- /dev/null
+++ b/docs/changelog/115359.yaml
@@ -0,0 +1,6 @@
+pr: 115359
+summary: Adding support for simulate ingest mapping adddition for indices with mappings
+  that do not come from templates
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/115383.yaml b/docs/changelog/115383.yaml
new file mode 100644
index 0000000000000..19eadd41c0726
--- /dev/null
+++ b/docs/changelog/115383.yaml
@@ -0,0 +1,5 @@
+pr: 115383
+summary: Only publish desired balance gauges on master
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/changelog/115393.yaml b/docs/changelog/115393.yaml
new file mode 100644
index 0000000000000..5cf4e5f64ab34
--- /dev/null
+++ b/docs/changelog/115393.yaml
@@ -0,0 +1,18 @@
+pr: 115393
+summary: Remove deprecated local attribute from alias APIs
+area: Indices APIs
+type: breaking
+issues: []
+breaking:
+  title: Remove deprecated local attribute from alias APIs
+  area: REST API
+  details: >-
+    The following APIs no longer accept the `?local` query parameter:
+    `GET /_alias`, `GET /_aliases`, `GET /_alias/{name}`,
+    `HEAD /_alias/{name}`, `GET /{index}/_alias`, `HEAD /{index}/_alias`,
+    `GET /{index}/_alias/{name}`, `HEAD /{index}/_alias/{name}`,
+    `GET /_cat/aliases`, and `GET /_cat/aliases/{alias}`. This parameter
+    has been deprecated and ignored since version 8.12.
+  impact: >-
+    Cease usage of the `?local` query parameter when calling the listed APIs.
+  notable: false
diff --git a/docs/changelog/115399.yaml b/docs/changelog/115399.yaml
new file mode 100644
index 0000000000000..9f69657a5d167
--- /dev/null
+++ b/docs/changelog/115399.yaml
@@ -0,0 +1,29 @@
+pr: 115399
+summary: Adding breaking change entry for retrievers
+area: Search
+type: breaking
+issues: []
+breaking:
+  title: Reworking RRF retriever to be evaluated during rewrite phase
+  area: REST API
+  details: |-
+    In this release (8.16), we have introduced major changes to the retrievers framework 
+    and how they can be evaluated, focusing mainly on compound retrievers 
+    like `rrf` and `text_similarity_reranker`, which allowed us to support full 
+    composability (i.e. any retriever can be nested under any compound retriever), 
+    as well as supporting additional search features like collapsing, explaining, 
+    aggregations, and highlighting.
+    
+    To ensure consistency, and given that this rework is not available until 8.16, 
+    `rrf` and `text_similarity_reranker`  retriever queries would now  
+    throw an exception in a mixed cluster scenario, where there are nodes 
+    both in current or later (i.e. >= 8.16) and previous ( <= 8.15) versions.
+    
+    As part of the rework, we have also removed the `_rank` property from 
+    the responses of an `rrf` retriever.
+  impact: |-
+    - Users will not be able to use the `rrf` and `text_similarity_reranker` retrievers in a mixed cluster scenario
+    with previous releases (i.e. prior to 8.16), and the request will throw an `IllegalArgumentException`.
+    - `_rank` has now been removed from the output of the `rrf` retrievers so trying to directly parse the field
+    will throw an exception
+  notable: false
diff --git a/docs/changelog/115404.yaml b/docs/changelog/115404.yaml
new file mode 100644
index 0000000000000..e443b152955f3
--- /dev/null
+++ b/docs/changelog/115404.yaml
@@ -0,0 +1,5 @@
+pr: 115404
+summary: Fix NPE in Get Deployment Stats
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/115414.yaml b/docs/changelog/115414.yaml
new file mode 100644
index 0000000000000..7475b765bb30e
--- /dev/null
+++ b/docs/changelog/115414.yaml
@@ -0,0 +1,9 @@
+pr: 115414
+summary: Mitigate IOSession timeouts
+area: Machine Learning
+type: bug
+issues:
+ - 114385
+ - 114327
+ - 114105
+ - 114232
diff --git a/docs/changelog/115429.yaml b/docs/changelog/115429.yaml
new file mode 100644
index 0000000000000..ddf3c69183000
--- /dev/null
+++ b/docs/changelog/115429.yaml
@@ -0,0 +1,5 @@
+pr: 115429
+summary: "[otel-data] Add more kubernetes aliases"
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/115430.yaml b/docs/changelog/115430.yaml
new file mode 100644
index 0000000000000..c2903f7751012
--- /dev/null
+++ b/docs/changelog/115430.yaml
@@ -0,0 +1,5 @@
+pr: 115430
+summary: Prevent NPE if model assignment is removed while waiting to start
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/115459.yaml b/docs/changelog/115459.yaml
new file mode 100644
index 0000000000000..b20a8f765c084
--- /dev/null
+++ b/docs/changelog/115459.yaml
@@ -0,0 +1,5 @@
+pr: 115459
+summary: Guard blob store local directory creation with `doPrivileged`
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/115594.yaml b/docs/changelog/115594.yaml
new file mode 100644
index 0000000000000..91a6089dfb3ce
--- /dev/null
+++ b/docs/changelog/115594.yaml
@@ -0,0 +1,6 @@
+pr: 115594
+summary: Update `BlobCacheBufferedIndexInput::readVLong` to correctly handle negative
+  long values
+area: Search
+type: bug
+issues: []
diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc
index 02980a4ed8a8c..0d3e76f71d238 100644
--- a/docs/plugins/analysis-nori.asciidoc
+++ b/docs/plugins/analysis-nori.asciidoc
@@ -244,11 +244,11 @@ Which responds with:
           "end_offset": 3,
           "type": "word",
           "position": 1,
-          "leftPOS": "J(Ending Particle)",
+          "leftPOS": "JKS(Subject case marker)",
           "morphemes": null,
           "posType": "MORPHEME",
           "reading": null,
-          "rightPOS": "J(Ending Particle)"
+          "rightPOS": "JKS(Subject case marker)"
         },
         {
           "token": "깊",
@@ -268,11 +268,11 @@ Which responds with:
           "end_offset": 6,
           "type": "word",
           "position": 3,
-          "leftPOS": "E(Verbal endings)",
+          "leftPOS": "ETM(Adnominal form transformative ending)",
           "morphemes": null,
           "posType": "MORPHEME",
           "reading": null,
-          "rightPOS": "E(Verbal endings)"
+          "rightPOS": "ETM(Adnominal form transformative ending)"
         },
         {
           "token": "나무",
@@ -292,11 +292,11 @@ Which responds with:
           "end_offset": 10,
           "type": "word",
           "position": 5,
-          "leftPOS": "J(Ending Particle)",
+          "leftPOS": "JX(Auxiliary postpositional particle)",
           "morphemes": null,
           "posType": "MORPHEME",
           "reading": null,
-          "rightPOS": "J(Ending Particle)"
+          "rightPOS": "JX(Auxiliary postpositional particle)"
         }
       ]
     },
diff --git a/docs/plugins/mapper-annotated-text.asciidoc b/docs/plugins/mapper-annotated-text.asciidoc
index e4141e98a2285..956b6bedffff1 100644
--- a/docs/plugins/mapper-annotated-text.asciidoc
+++ b/docs/plugins/mapper-annotated-text.asciidoc
@@ -155,11 +155,6 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`annotated_text` fields support {ref}/mapping-source-field.html#synthetic-source[synthetic `_source`] if they have
-a {ref}/keyword.html#keyword-synthetic-source[`keyword`] sub-field that supports synthetic
-`_source` or if the `annotated_text` field sets `store` to `true`. Either way, it may
-not have {ref}/copy-to.html[`copy_to`].
-
 If using a sub-`keyword` field then the values are sorted in the same way as
 a `keyword` field's values are sorted. By default, that means sorted with
 duplicates removed. So:
@@ -167,8 +162,16 @@ duplicates removed. So:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": {
         "type": "annotated_text",
@@ -215,8 +218,16 @@ are preserved.
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": { "type": "annotated_text", "store": true }
     }
diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
index 5273537389e3d..881970787f5a6 100644
--- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
@@ -1430,7 +1430,8 @@ PUT /persian_example
             "decimal_digit",
             "arabic_normalization",
             "persian_normalization",
-            "persian_stop"
+            "persian_stop",
+            "persian_stem"
           ]
         }
       }
diff --git a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
index 4cd088935af19..d9e2120afe6d1 100644
--- a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
@@ -173,7 +173,6 @@ http://bvg.udc.es/recursos_lingua/stemming.jsp[`minimal_galician`] (Plural step
 German::
 https://dl.acm.org/citation.cfm?id=1141523[*`light_german`*],
 https://snowballstem.org/algorithms/german/stemmer.html[`german`],
-https://snowballstem.org/algorithms/german2/stemmer.html[`german2`],
 http://members.unine.ch/jacques.savoy/clef/morpho.pdf[`minimal_german`]
 
 Greek::
diff --git a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
index 2cf01b77d57ab..5f98807387280 100644
--- a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
@@ -40,14 +40,14 @@ POST _analyze
       "start_offset": 0,
       "end_offset": 8,
       "type": "word",
-      "position": 0
+      "position": 1
     },
     {
       "token": "/one/two/three",
       "start_offset": 0,
       "end_offset": 14,
       "type": "word",
-      "position": 0
+      "position": 2
     }
   ]
 }
@@ -144,14 +144,14 @@ POST my-index-000001/_analyze
       "start_offset": 7,
       "end_offset": 18,
       "type": "word",
-      "position": 0
+      "position": 1
     },
     {
       "token": "/three/four/five",
       "start_offset": 7,
       "end_offset": 23,
       "type": "word",
-      "position": 0
+      "position": 2
     }
   ]
 }
@@ -178,14 +178,14 @@ If we were to set `reverse` to `true`, it would produce the following:
 [[analysis-pathhierarchy-tokenizer-detailed-examples]]
 === Detailed examples
 
-A common use-case for the `path_hierarchy` tokenizer is filtering results by 
-file paths. If indexing a file path along with the data, the use of the 
-`path_hierarchy` tokenizer to analyze the path allows filtering the results 
+A common use-case for the `path_hierarchy` tokenizer is filtering results by
+file paths. If indexing a file path along with the data, the use of the
+`path_hierarchy` tokenizer to analyze the path allows filtering the results
 by different parts of the file path string.
 
 
 This example configures an index to have two custom analyzers and applies
-those analyzers to multifields of the `file_path` text field that will 
+those analyzers to multifields of the `file_path` text field that will
 store filenames. One of the two analyzers uses reverse tokenization.
 Some sample documents are then indexed to represent some file paths
 for photos inside photo folders of two different users.
@@ -264,8 +264,8 @@ POST file-path-test/_doc/5
 --------------------------------------------------
 
 
-A search for a particular file path string against the text field matches all 
-the example documents, with Bob's documents ranking highest due to `bob` also 
+A search for a particular file path string against the text field matches all
+the example documents, with Bob's documents ranking highest due to `bob` also
 being one of the terms created by the standard analyzer boosting relevance for
 Bob's documents.
 
@@ -301,7 +301,7 @@ GET file-path-test/_search
 With the reverse parameter for this tokenizer, it's also possible to match
 from the other end of the file path, such as individual file names or a deep
 level subdirectory. The following example shows a search for all files named
-`my_photo1.jpg` within any directory via the `file_path.tree_reversed` field 
+`my_photo1.jpg` within any directory via the `file_path.tree_reversed` field
 configured to use the reverse parameter in the mapping.
 
 
@@ -342,7 +342,7 @@ POST file-path-test/_analyze
 
 
 It's also useful to be able to filter with file paths when combined with other
-types of searches, such as this example looking for any files paths with `16` 
+types of searches, such as this example looking for any files paths with `16`
 that also must be in Alice's photo directory.
 
 [source,console]
diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc
index 72f949bf11e50..41ac279d3b2f5 100644
--- a/docs/reference/cat/alias.asciidoc
+++ b/docs/reference/cat/alias.asciidoc
@@ -6,8 +6,8 @@
 
 [IMPORTANT]
 ====
-cat APIs are only intended for human consumption using the command line or the 
-{kib} console. They are _not_ intended for use by applications. For application 
+cat APIs are only intended for human consumption using the command line or the
+{kib} console. They are _not_ intended for use by applications. For application
 consumption, use the <<aliases,aliases API>>.
 ====
 
@@ -45,8 +45,6 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help]
 
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]
-
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v]
@@ -104,6 +102,6 @@ alias4 test1 -      2            1,2            -
 This response shows that `alias2` has configured a filter, and specific routing
 configurations in `alias3` and `alias4`.
 
-If you only want to get information about specific aliases, you can specify 
-the aliases in comma-delimited format as a URL parameter, e.g., 
+If you only want to get information about specific aliases, you can specify
+the aliases in comma-delimited format as a URL parameter, e.g.,
 /_cat/aliases/alias1,alias2.
diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc
index cf1cc9f825cb2..b8dda01c2eae0 100644
--- a/docs/reference/cat/indices.asciidoc
+++ b/docs/reference/cat/indices.asciidoc
@@ -6,8 +6,8 @@
 
 [IMPORTANT]
 ====
-cat APIs are only intended for human consumption using the command line or {kib} 
-console. They are _not_ intended for use by applications. For application 
+cat APIs are only intended for human consumption using the command line or {kib}
+console. They are _not_ intended for use by applications. For application
 consumption, use the <<indices-get-index,get index API>>.
 ====
 
@@ -50,6 +50,10 @@ indexing and search. As a result, all document counts include hidden
 To get an accurate count of {es} documents, use the <<cat-count,cat count>> or
 <<search-count,count>> APIs.
 
+Note that information such as document count, deleted document count and store size are not shown for
+indices restored from <<snapshots-source-only-repository,source-only snapshots>> since these indices
+do not contain the relevant data structures to retrieve this information from.
+
 
 [[cat-indices-api-path-params]]
 ==== {api-path-parms-title}
diff --git a/docs/reference/cat/trainedmodel.asciidoc b/docs/reference/cat/trainedmodel.asciidoc
index 45c87038f5d64..5b20a0b6e842f 100644
--- a/docs/reference/cat/trainedmodel.asciidoc
+++ b/docs/reference/cat/trainedmodel.asciidoc
@@ -116,7 +116,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v]
 
 [source,console]
 --------------------------------------------------
-GET _cat/ml/trained_models?h=c,o,l,ct,v&v=ture
+GET _cat/ml/trained_models?h=c,o,l,ct,v&v=true
 --------------------------------------------------
 // TEST[skip:kibana sample data]
 
diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
index 6aa0c6110277c..bbbea192f0f86 100644
--- a/docs/reference/cluster/allocation-explain.asciidoc
+++ b/docs/reference/cluster/allocation-explain.asciidoc
@@ -159,6 +159,7 @@ node.
 <5> The decider which led to the `no` decision for the node.
 <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. In this example, a newly created index has <<indices-get-settings,an index setting>> that requires that it only be allocated to a node named `nonexistent_node`, which does not exist, so the index is unable to allocate.
 
+[[maximum-number-of-retries-exceeded]]
 ====== Maximum number of retries exceeded
 
 The following response contains an allocation explanation for an unassigned
@@ -195,7 +196,7 @@ primary shard that has reached the maximum number of allocation retry attempts.
         {
           "decider": "max_retry",
           "decision" : "NO",
-          "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]"
+          "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [POST /_cluster/reroute?retry_failed] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]"
         }
       ]
     }
@@ -203,9 +204,11 @@ primary shard that has reached the maximum number of allocation retry attempts.
 }
 ----
 // NOTCONSOLE
-
-If decider message indicates a transient allocation issue, use
-the <<cluster-reroute,cluster reroute>> API to retry allocation.
+When Elasticsearch is unable to allocate a shard, it will attempt to retry allocation up to
+the maximum number of retries allowed. After this, Elasticsearch will stop attempting to
+allocate the shard in order to prevent infinite retries which may impact cluster
+performance. Run the <<cluster-reroute,cluster reroute>> API to retry allocation, which
+will allocate the shard if the issue preventing allocation has been resolved.
 
 [[no-valid-shard-copy]]
 ====== No valid shard copy
diff --git a/docs/reference/connector/docs/connectors-content-extraction.asciidoc b/docs/reference/connector/docs/connectors-content-extraction.asciidoc
index b785d62f0f553..5d2a9550a7c3c 100644
--- a/docs/reference/connector/docs/connectors-content-extraction.asciidoc
+++ b/docs/reference/connector/docs/connectors-content-extraction.asciidoc
@@ -90,7 +90,7 @@ include::_connectors-list-local-content-extraction.asciidoc[]
 Self-hosted content extraction is handled by a *separate* extraction service.
 
 The versions for the extraction service do not align with the Elastic stack.
-For version `8.11.x`, you should use extraction service version `0.3.x`.
+For versions after `8.11.x` (including {version}), you should use extraction service version `0.3.x`.
 
 You can run the service with the following command:
 
diff --git a/docs/reference/data-streams/logs.asciidoc b/docs/reference/data-streams/logs.asciidoc
index e870289bcf7be..6bb98684544a3 100644
--- a/docs/reference/data-streams/logs.asciidoc
+++ b/docs/reference/data-streams/logs.asciidoc
@@ -8,14 +8,6 @@ A logs data stream is a data stream type that stores log data more efficiently.
 In benchmarks, log data stored in a logs data stream used ~2.5 times less disk space than a regular data
 stream. The exact impact will vary depending on your data set.
 
-The following features are enabled in a logs data stream:
-
-* <<synthetic-source,Synthetic source>>, which omits storing the `_source` field. When the document source is requested, it is synthesized from document fields upon retrieval.
-
-* Index sorting. This yields a lower storage footprint. By default indices are sorted by `host.name` and `@timestamp` fields at index time.
-
-* More space efficient compression for fields with <<doc-values,`doc_values`>> enabled.
-
 [discrete]
 [[how-to-use-logsds]]
 === Create a logs data stream
@@ -50,3 +42,175 @@ DELETE _index_template/my-index-template
 ----
 // TEST[continued]
 ////
+
+[[logsdb-default-settings]]
+
+[discrete]
+[[logsdb-synthtic-source]]
+=== Synthetic source
+
+By default, `logsdb` mode uses <<synthetic-source,synthetic source>>, which omits storing the original `_source`
+field and synthesizes it from doc values or stored fields upon document retrieval. Synthetic source comes with a few
+restrictions which you can read more about in the <<synthetic-source,documentation>> section dedicated to it.
+
+NOTE: When dealing with multi-value fields, the `index.mapping.synthetic_source_keep` setting controls how field values
+are preserved for <<synthetic-source,synthetic source>> reconstruction. In `logsdb`, the default value is `arrays`,
+which retains both duplicate values and the order of entries but not necessarily the exact structure when it comes to
+array elements or objects. Preserving duplicates and ordering could be critical for some log fields. This could be the
+case, for instance, for DNS A records, HTTP headers, or log entries that represent sequential or repeated events.
+
+For more details on this setting and ways to refine or bypass it, check out <<synthetic-source-keep, this section>>.
+
+[discrete]
+[[logsdb-sort-settings]]
+=== Index sort settings
+
+The following settings are applied by default when using the `logsdb` mode for index sorting:
+
+* `index.sort.field`: `["host.name", "@timestamp"]`
+  In `logsdb` mode, indices are sorted by `host.name` and `@timestamp` fields by default. For data streams, the
+  `@timestamp` field is automatically injected if it is not present.
+
+* `index.sort.order`: `["desc", "desc"]`
+  The default sort order for both fields is descending (`desc`), prioritizing the latest data.
+
+* `index.sort.mode`: `["min", "min"]`
+  The default sort mode is `min`, ensuring that indices are sorted by the minimum value of multi-value fields.
+
+* `index.sort.missing`: `["_first", "_first"]`
+  Missing values are sorted to appear first (`_first`) in `logsdb` index mode.
+
+`logsdb` index mode allows users to override the default sort settings. For instance, users can specify their own fields
+and order for sorting by modifying the `index.sort.field` and `index.sort.order`.
+
+When using default sort settings, the `host.name` field is automatically injected into the mappings of the
+index as a `keyword` field to ensure that sorting can be applied. This guarantees that logs are efficiently sorted and
+retrieved based on the `host.name` and `@timestamp` fields.
+
+NOTE: If `subobjects` is set to `true` (which is the default), the `host.name` field will be mapped as an object field
+named `host`, containing a `name` child field of type `keyword`. On the other hand, if `subobjects` is set to `false`,
+a single `host.name` field will be mapped as a `keyword` field.
+
+Once an index is created, the sort settings are immutable and cannot be modified. To apply different sort settings,
+a new index must be created with the desired configuration. For data streams, this can be achieved by means of an index
+rollover after updating relevant (component) templates.
+
+If the default sort settings are not suitable for your use case, consider modifying them. Keep in mind that sort
+settings can influence indexing throughput, query latency, and may affect compression efficiency due to the way data
+is organized after sorting. For more details, refer to our documentation on
+<<index-modules-index-sorting,index sorting>>.
+
+NOTE: For <<data-streams, data streams>>, the `@timestamp` field is automatically injected if not already present.
+However, if custom sort settings are applied, the `@timestamp` field is injected into the mappings, but it is not
+automatically added to the list of sort fields.
+
+[discrete]
+[[logsdb-specialized-codecs]]
+=== Specialized codecs
+
+`logsdb` index mode uses the `best_compression` <<index-codec,codec>> by default, which applies {wikipedia}/Zstd[ZSTD]
+compression to stored fields. Users are allowed to override it and switch to the `default` codec for faster compression
+at the expense of slightly larger storage footprint.
+
+`logsdb` index mode also adopts specialized codecs for numeric doc values that are crafted to optimize storage usage.
+Users can rely on these specialized codecs being applied by default when using `logsdb` index mode.
+
+Doc values encoding for numeric fields in `logsdb` follows a static sequence of codecs, applying each one in the
+following order: delta encoding, offset encoding, Greatest Common Divisor GCD encoding, and finally Frame Of Reference
+(FOR) encoding. The decision to apply each encoding is based on heuristics determined by the data distribution.
+For example, before applying delta encoding, the algorithm checks if the data is monotonically non-decreasing or
+non-increasing. If the data fits this pattern, delta encoding is applied; otherwise, the next encoding is considered.
+
+The encoding is specific to each Lucene segment and is also re-applied at segment merging time. The merged Lucene segment
+may use a different encoding compared to the original Lucene segments, based on the characteristics of the merged data.
+
+The following methods are applied sequentially:
+
+* **Delta encoding**:
+  a compression method that stores the difference between consecutive values instead of the actual values.
+
+* **Offset encoding**:
+  a compression method that stores the difference from a base value rather than between consecutive values.
+
+* **Greatest Common Divisor (GCD) encoding**:
+  a compression method that finds the greatest common divisor of a set of values and stores the differences
+  as multiples of the GCD.
+
+* **Frame Of Reference (FOR) encoding**:
+  a compression method that determines the smallest number of bits required to encode a block of values and uses
+  bit-packing to fit such values into larger 64-bit blocks.
+
+For keyword fields, **Run Length Encoding (RLE)** is applied to the ordinals, which represent positions in the Lucene
+segment-level keyword dictionary. This compression is used when multiple consecutive documents share the same keyword.
+
+[discrete]
+[[logsdb-ignored-settings]]
+=== `ignore_malformed`, `ignore_above`, `ignore_dynamic_beyond_limit`
+
+By default, `logsdb` index mode sets `ignore_malformed` to `true`. This setting allows documents with malformed fields
+to be indexed without causing indexing failures, ensuring that log data ingestion continues smoothly even when some
+fields contain invalid or improperly formatted data.
+
+Users can override this setting by setting `index.mapping.ignore_malformed` to `false`. However, this is not recommended
+as it might result in documents with malformed fields being rejected and not indexed at all.
+
+In `logsdb` index mode, the `index.mapping.ignore_above` setting is applied by default at the index level to ensure
+efficient storage and indexing of large keyword fields.The index-level default for `ignore_above` is set to 8191
+**characters**. If using UTF-8 encoding, this results in a limit of 32764 bytes, depending on character encoding.
+The mapping-level `ignore_above` setting still takes precedence. If a specific field has an `ignore_above` value
+defined in its mapping, that value will override the index-level `index.mapping.ignore_above` value. This default
+behavior helps to optimize indexing performance by preventing excessively large string values from being indexed, while
+still allowing users to customize the limit, overriding it at the mapping level or changing the index level default
+setting.
+
+In `logsdb` index mode, the setting `index.mapping.total_fields.ignore_dynamic_beyond_limit` is set to `true` by
+default. This allows dynamically mapped fields to be added on top of statically defined fields without causing document
+rejection, even after the total number of fields exceeds the limit defined by `index.mapping.total_fields.limit`. The
+`index.mapping.total_fields.limit` setting specifies the maximum number of fields an index can have (static, dynamic
+and runtime). When the limit is reached, new dynamically mapped fields will be ignored instead of failing the document
+indexing, ensuring continued log ingestion without errors.
+
+NOTE: When automatically injected, `host.name` and `@timestamp` contribute to the limit of mapped fields. When
+`host.name` is mapped with `subobjects: true` it consists of two fields. When `host.name` is mapped with
+`subobjects: false` it only consists of one field.
+
+[discrete]
+[[logsdb-nodocvalue-fields]]
+=== Fields without doc values
+
+When `logsdb` index mode uses synthetic `_source`, and `doc_values` are disabled for a field in the mapping,
+Elasticsearch may set the `store` setting to `true` for that field as a last resort option to ensure that the field's
+data is still available for reconstructing the document’s source when retrieving it via
+<<synthetic-source,synthetic source>>.
+
+For example, this happens with text fields when `store` is `false` and there is no suitable multi-field available to
+reconstruct the original value in <<synthetic-source,synthetic source>>.
+
+This automatic adjustment allows synthetic source to work correctly, even when doc values are not enabled for certain
+fields.
+
+[discrete]
+[[logsdb-settings-summary]]
+=== LogsDB settings summary
+
+The following is a summary of key settings that apply when using `logsdb` index mode in Elasticsearch:
+
+* **`index.mode`**: `"logsdb"`
+
+* **`index.mapping.synthetic_source_keep`**: `"arrays"`
+
+* **`index.sort.field`**: `["host.name", "@timestamp"]`
+
+* **`index.sort.order`**: `["desc", "desc"]`
+
+* **`index.sort.mode`**: `["min", "min"]`
+
+* **`index.sort.missing`**: `["_first", "_first"]`
+
+* **`index.codec`**: `"best_compression"`
+
+* **`index.mapping.ignore_malformed`**: `true`
+
+* **`index.mapping.ignore_above`**: `8191`
+
+* **`index.mapping.total_fields.ignore_dynamic_beyond_limit`**: `true`
diff --git a/docs/reference/data-streams/promote-data-stream-api.asciidoc b/docs/reference/data-streams/promote-data-stream-api.asciidoc
index 111c7a2256f8a..5ba9c4d9fad0e 100644
--- a/docs/reference/data-streams/promote-data-stream-api.asciidoc
+++ b/docs/reference/data-streams/promote-data-stream-api.asciidoc
@@ -18,6 +18,10 @@ available, the data stream in the local cluster can be promoted
 to a regular data stream, which allows these data streams to
 be rolled over in the local cluster.
 
+NOTE: When promoting a data stream, ensure the local cluster has a data stream enabled index template that matches the data stream.
+If this is missing, the data stream will not be able to roll over until a matching index template is created.
+This will affect the lifecycle management of the data stream and interfere with the data stream size and retention.
+
 [source,console]
 ----
 POST /_data_stream/_promote/my-data-stream
diff --git a/docs/reference/data-streams/set-up-tsds.asciidoc b/docs/reference/data-streams/set-up-tsds.asciidoc
index 3a483ac351180..d082a9c4eebeb 100644
--- a/docs/reference/data-streams/set-up-tsds.asciidoc
+++ b/docs/reference/data-streams/set-up-tsds.asciidoc
@@ -121,7 +121,8 @@ naming scheme].
 * Specify a mapping that defines your dimensions and metrics:
 
 ** One or more <<time-series-dimension,dimension fields>> with a `time_series_dimension` value of `true`.
-   At least one of these dimensions must be a plain `keyword` field.
+   Alternatively, one or more <<passthrough-dimensions, pass-through>> fields configured as dimension containers,
+   provided that they will contain at least one sub-field (mapped statically or dynamically).
 
 ** One or more <<time-series-metric,metric fields>>, marked using the `time_series_metric` mapping parameter.
 
@@ -203,10 +204,9 @@ DELETE _ilm/policy/my-weather-sensor-lifecycle-policy
 Documents in a TSDS must include:
 
 * A `@timestamp` field
-* One or more dimension fields. At least one dimension must be a `keyword` field
-that matches the `index.routing_path` index setting, if specified. If not specified
-explicitly, `index.routing_path` is set automatically to whichever mappings have
- `time_series_dimension` set to `true`.
+* One or more dimension fields. At least one dimension must match the `index.routing_path` index setting,
+if specified. If not specified explicitly, `index.routing_path` is set automatically to whichever mappings have
+`time_series_dimension` set to `true`.
 
 To automatically create your TSDS, submit an indexing request that
 targets the TSDS's name. This name must match one of your index template's
@@ -285,13 +285,12 @@ POST metrics-weather_sensors-dev/_rollover
 
 Configuring a TSDS via an index template that uses component templates is a bit more complicated.
 Typically with component templates mappings and settings get scattered across multiple component templates.
-When configuring the `index.mode` setting in a component template, the `index.routing_path` setting needs to
-be defined in the same component template. Additionally the fields mentioned in the `index.routing_path`
-also need to be defined in the same component template with the `time_series_dimension` attribute enabled.
+If the `index.routing_path` is defined, the fields it references need to be defined in the same component
+template with the `time_series_dimension` attribute enabled.
 
-The reasons for this is that each component template needs to be valid on its own and the time series index mode
-requires the `index.routing_path` setting. When configuring the  `index.mode` setting in an index template, the `index.routing_path` setting is configured automatically. It is derived  from
-the field mappings with `time_series_dimension` attribute enabled.
+The reasons for this is that each component template needs to be valid on its own. When configuring the
+`index.mode` setting in an index template, the `index.routing_path` setting is configured automatically.
+It is derived from the field mappings with `time_series_dimension` attribute enabled.
 
 [discrete]
 [[set-up-tsds-whats-next]]
diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc
index 01573658c33d0..461c0a1272e96 100644
--- a/docs/reference/data-streams/tsds.asciidoc
+++ b/docs/reference/data-streams/tsds.asciidoc
@@ -109,7 +109,10 @@ parameter:
 * <<number,`unsigned_long`>>
 * <<boolean,`boolean`>>
 
-For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. For details refer to <<flattened-params,`flattened`>>.
+For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions.
+For details refer to <<flattened-params,`flattened`>>.
+
+Dimension definitions can be simplified through <<passthrough-dimensions, pass-through>> fields.
 
 [discrete]
 [[time-series-metric]]
@@ -294,12 +297,15 @@ When you create the matching index template for a TSDS, you must specify one or
 more dimensions in the `index.routing_path` setting. Each document in a TSDS
 must contain one or more dimensions that match the `index.routing_path` setting.
 
-Dimensions in the `index.routing_path` setting must be plain `keyword` fields.
 The `index.routing_path` setting accepts wildcard patterns (for example `dim.*`)
 and can dynamically match new fields. However, {es} will reject any mapping
-updates that add scripted, runtime, or non-dimension, non-`keyword` fields that
+updates that add scripted, runtime, or non-dimension fields that
 match the `index.routing_path` value.
 
+<<passthrough-dimensions, Pass-through>> fields may be configured
+as dimension containers. In this case, their sub-fields get included to the
+routing path automatically.
+
 TSDS documents don't support a custom `_routing` value. Similarly, you can't
 require a `_routing` value in mappings for a TSDS.
 
diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc
index cfcb5de73602c..db266fafde9d6 100644
--- a/docs/reference/esql/esql-across-clusters.asciidoc
+++ b/docs/reference/esql/esql-across-clusters.asciidoc
@@ -188,9 +188,10 @@ FROM *:my-index-000001
 [[ccq-cluster-details]]
 ==== Cross-cluster metadata
 
-ES|QL {ccs} responses include metadata about the search on each cluster when the response format is JSON.
+Using the `"include_ccs_metadata": true` option, users can request that
+ES|QL {ccs} responses include metadata about the search on each cluster (when the response format is JSON).
 Here we show an example using the async search endpoint. {ccs-cap} metadata is also present in the synchronous
-search endpoint.
+search endpoint response when requested.
 
 [source,console]
 ----
@@ -200,7 +201,8 @@ POST /_query/async?format=json
     FROM my-index-000001,cluster_one:my-index-000001,cluster_two:my-index*
     | STATS COUNT(http.response.status_code) BY user.id
     | LIMIT 2
-  """
+  """,
+  "include_ccs_metadata": true
 }
 ----
 // TEST[setup:my_index]
@@ -238,7 +240,7 @@ Which returns:
       "(local)": { <4>
         "status": "successful",
         "indices": "blogs",
-        "took": 36,  <5>
+        "took": 41,  <5>
         "_shards": { <6>
           "total": 13,
           "successful": 13,
@@ -260,7 +262,7 @@ Which returns:
       "cluster_two": {
         "status": "successful",
         "indices": "cluster_two:my-index*",
-        "took": 41,
+        "took": 40,
         "_shards": {
           "total": 18,
           "successful": 18,
@@ -286,7 +288,7 @@ it is identified as "(local)".
 <5> How long (in milliseconds) the search took on each cluster. This can be useful to determine
 which clusters have slower response times than others.
 <6> The shard details for the search on that cluster, including a count of shards that were
-skipped due to the can-match phase. Shards are skipped when they cannot have any matching data
+skipped due to the can-match phase results. Shards are skipped when they cannot have any matching data
 and therefore are not included in the full ES|QL query.
 
 
@@ -294,9 +296,6 @@ The cross-cluster metadata can be used to determine whether any data came back f
 For instance, in the query below, the wildcard expression for `cluster-two` did not resolve
 to a concrete index (or indices). The cluster is, therefore, marked as 'skipped' and the total
 number of shards searched is set to zero.
-Since the other cluster did have a matching index, the search did not return an error, but
-instead returned all the matching data it could find.
-
 
 [source,console]
 ----
@@ -306,7 +305,8 @@ POST /_query/async?format=json
     FROM cluster_one:my-index*,cluster_two:logs*
     | STATS COUNT(http.response.status_code) BY user.id
     | LIMIT 2
-  """
+  """,
+  "include_ccs_metadata": true
 }
 ----
 // TEST[continued]
diff --git a/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc b/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc
index 87748fee4f202..e626e058a4e56 100644
--- a/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc
+++ b/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc
@@ -40,7 +40,7 @@ delimiter-based pattern, and extracts the specified keys as columns.
 For example, the following pattern:
 [source,txt]
 ----
-%{clientip} [%{@timestamp}] %{status} 
+%{clientip} [%{@timestamp}] %{status}
 ----
 
 matches a log line of this format:
@@ -76,8 +76,8 @@ ignore certain fields, append fields, skip over padding, etc.
 ===== Terminology
 
 dissect pattern::
-the set of fields and delimiters describing the textual 
-format. Also known as a dissection. 
+the set of fields and delimiters describing the textual
+format. Also known as a dissection.
 The dissection is described using a set of `%{}` sections:
 `%{a} - %{b} - %{c}`
 
@@ -91,14 +91,14 @@ Any set of characters other than `%{`, `'not }'`, or `}` is a delimiter.
 key::
 +
 --
-the text between the `%{` and `}`, exclusive of the `?`, `+`, `&` prefixes 
-and the ordinal suffix. 
+the text between the `%{` and `}`, exclusive of the `?`, `+`, `&` prefixes
+and the ordinal suffix.
 
 Examples:
 
-* `%{?aaa}` - the key is `aaa` 
-* `%{+bbb/3}` - the key is `bbb` 
-* `%{&ccc}` - the key is `ccc` 
+* `%{?aaa}` - the key is `aaa`
+* `%{+bbb/3}` - the key is `bbb`
+* `%{&ccc}` - the key is `ccc`
 --
 
 [[esql-dissect-examples]]
@@ -218,7 +218,7 @@ Putting it together as an {esql} query:
 
 [source.merge.styled,esql]
 ----
-include::{esql-specs}/docs.csv-spec[tag=grokWithEscape]
+include::{esql-specs}/docs.csv-spec[tag=grokWithEscapeTripleQuotes]
 ----
 
 `GROK` adds the following columns to the input table:
@@ -239,15 +239,24 @@ with a `\`. For example, in the earlier pattern:
 %{IP:ip} \[%{TIMESTAMP_ISO8601:@timestamp}\] %{GREEDYDATA:status}
 ----
 
-In {esql} queries, the backslash character itself is a special character that
+In {esql} queries, when using single quotes for strings, the backslash character itself is a special character that
 needs to be escaped with another `\`. For this example, the corresponding {esql}
 query becomes:
 [source.merge.styled,esql]
 ----
 include::{esql-specs}/docs.csv-spec[tag=grokWithEscape]
 ----
+
+For this reason, in general it is more convenient to use triple quotes `"""` for GROK patterns,
+that do not require escaping for backslash.
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/docs.csv-spec[tag=grokWithEscapeTripleQuotes]
+----
 ====
 
+
 [[esql-grok-patterns]]
 ===== Grok patterns
 
@@ -318,4 +327,4 @@ as the `GROK` command.
 The `GROK` command does not support configuring <<custom-patterns,custom
 patterns>>, or <<trace-match,multiple patterns>>. The `GROK` command is not
 subject to <<grok-watchdog,Grok watchdog settings>>.
-// end::grok-limitations[]
\ No newline at end of file
+// end::grok-limitations[]
diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc
index d1db21043a5b5..b1582721ad0e0 100644
--- a/docs/reference/esql/esql-query-api.asciidoc
+++ b/docs/reference/esql/esql-query-api.asciidoc
@@ -67,6 +67,11 @@ precedence.
 `false`. The API only supports this parameter for CBOR, JSON, SMILE, and YAML
 responses. See <<esql-rest-columnar>>.
 
+`include_ccs_metadata`::
+(Optional, boolean) If `true`, cross-cluster searches will include metadata about the query
+on each cluster. Defaults to `false`. The API only supports this parameter for CBOR, JSON, SMILE,
+and YAML responses. See <<ccq-cluster-details>>.
+
 `locale`::
 (Optional, string) Returns results (especially dates) formatted per the conventions of the locale.
 For syntax, refer to <<esql-locale-param>>.
@@ -85,6 +90,7 @@ https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN].
 `query`::
 (Required, string) {esql} query to run. For syntax, refer to <<esql-syntax>>.
 
+
 ifeval::["{release-state}"=="unreleased"]
 `table`::
 (Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
@@ -108,6 +114,13 @@ returned if `drop_null_columns` is sent with the request.
 (array of arrays)
 Values for the search results.
 
+`_clusters`::
+(object)
+Metadata about clusters involved in the execution of a cross-cluster query. Only returned (1) for
+cross-cluster searches and (2) when `include_ccs_metadata` is sent in the body and set to `true`
+and (3) when `format` of the response is set to JSON (the default), CBOR, SMILE, or YAML.
+See <<ccq-cluster-details>> for more information.
+
 `profile`::
 (object)
 Profile describing the execution of the query. Only returned if `profile` was sent in the body.
diff --git a/docs/reference/esql/functions/description/hypot.asciidoc b/docs/reference/esql/functions/description/hypot.asciidoc
new file mode 100644
index 0000000000000..5162f0d9ef98f
--- /dev/null
+++ b/docs/reference/esql/functions/description/hypot.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double. Hypotenuses of infinities are null.
diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc
new file mode 100644
index 0000000000000..2a27fe4814395
--- /dev/null
+++ b/docs/reference/esql/functions/description/match.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Performs a match query on the specified field. Returns true if the provided query matches the row.
diff --git a/docs/reference/esql/functions/examples/hypot.asciidoc b/docs/reference/esql/functions/examples/hypot.asciidoc
new file mode 100644
index 0000000000000..6dbcc62e8755e
--- /dev/null
+++ b/docs/reference/esql/functions/examples/hypot.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/math.csv-spec[tag=hypot]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/math.csv-spec[tag=hypot-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/match.asciidoc b/docs/reference/esql/functions/examples/match.asciidoc
new file mode 100644
index 0000000000000..3f31d68ea9abb
--- /dev/null
+++ b/docs/reference/esql/functions/examples/match.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/match-function.csv-spec[tag=match-with-field]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/match-function.csv-spec[tag=match-with-field-result]
+|===
+
diff --git a/docs/reference/esql/functions/kibana/definition/hypot.json b/docs/reference/esql/functions/kibana/definition/hypot.json
new file mode 100644
index 0000000000000..06971f07a3585
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/hypot.json
@@ -0,0 +1,301 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "hypot",
+  "description" : "Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double.\nHypotenuses of infinities are null.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number1",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "number2",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Numeric expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    }
+  ],
+  "examples" : [
+    "ROW a = 3.0, b = 4.0\n| EVAL c = HYPOT(a, b)"
+  ],
+  "preview" : false,
+  "snapshot_only" : false
+}
diff --git a/docs/reference/esql/functions/kibana/definition/like.json b/docs/reference/esql/functions/kibana/definition/like.json
index 97e84e0361fd2..f375c697bd60d 100644
--- a/docs/reference/esql/functions/kibana/definition/like.json
+++ b/docs/reference/esql/functions/kibana/definition/like.json
@@ -42,7 +42,7 @@
     }
   ],
   "examples" : [
-    "FROM employees\n| WHERE first_name LIKE \"?b*\"\n| KEEP first_name, last_name"
+    "FROM employees\n| WHERE first_name LIKE \"\"\"?b*\"\"\"\n| KEEP first_name, last_name"
   ],
   "preview" : false,
   "snapshot_only" : false
diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json
new file mode 100644
index 0000000000000..8a355360a790f
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/match.json
@@ -0,0 +1,85 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "match",
+  "description" : "Performs a match query on the specified field. Returns true if the provided query matches the row.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Field that the query will target."
+        },
+        {
+          "name" : "query",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Text you wish to find in the provided field."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Field that the query will target."
+        },
+        {
+          "name" : "query",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Text you wish to find in the provided field."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Field that the query will target."
+        },
+        {
+          "name" : "query",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Text you wish to find in the provided field."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Field that the query will target."
+        },
+        {
+          "name" : "query",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Text you wish to find in the provided field."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    }
+  ],
+  "examples" : [
+    "from books \n| where match(author, \"Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;"
+  ],
+  "preview" : true,
+  "snapshot_only" : false
+}
diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json
index 72be906cbae63..9823c3cff8923 100644
--- a/docs/reference/esql/functions/kibana/definition/qstr.json
+++ b/docs/reference/esql/functions/kibana/definition/qstr.json
@@ -33,5 +33,5 @@
     "from books \n| where qstr(\"author: Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;"
   ],
   "preview" : true,
-  "snapshot_only" : true
+  "snapshot_only" : false
 }
diff --git a/docs/reference/esql/functions/kibana/definition/rlike.json b/docs/reference/esql/functions/kibana/definition/rlike.json
index e442bb2c55050..7a328293383bb 100644
--- a/docs/reference/esql/functions/kibana/definition/rlike.json
+++ b/docs/reference/esql/functions/kibana/definition/rlike.json
@@ -42,7 +42,7 @@
     }
   ],
   "examples" : [
-    "FROM employees\n| WHERE first_name RLIKE \".leja.*\"\n| KEEP first_name, last_name"
+    "FROM employees\n| WHERE first_name RLIKE \"\"\".leja.*\"\"\"\n| KEEP first_name, last_name"
   ],
   "preview" : false,
   "snapshot_only" : false
diff --git a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
index bafbcf2bc2038..07ffe84444f02 100644
--- a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
+++ b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
@@ -5,5 +5,6 @@
   "description" : "Converts an input to a nanosecond-resolution date value (aka date_nanos).",
   "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z.  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.",
   "signatures" : [ ],
-  "preview" : true
+  "preview" : true,
+  "snapshot_only" : false
 }
diff --git a/docs/reference/esql/functions/kibana/docs/hypot.md b/docs/reference/esql/functions/kibana/docs/hypot.md
new file mode 100644
index 0000000000000..f0cbea6b88e55
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/hypot.md
@@ -0,0 +1,12 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### HYPOT
+Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double.
+Hypotenuses of infinities are null.
+
+```
+ROW a = 3.0, b = 4.0
+| EVAL c = HYPOT(a, b)
+```
diff --git a/docs/reference/esql/functions/kibana/docs/like.md b/docs/reference/esql/functions/kibana/docs/like.md
index 4c400bdc65479..ea2ac11b6f4b9 100644
--- a/docs/reference/esql/functions/kibana/docs/like.md
+++ b/docs/reference/esql/functions/kibana/docs/like.md
@@ -15,6 +15,6 @@ The following wildcard characters are supported:
 
 ```
 FROM employees
-| WHERE first_name LIKE "?b*"
+| WHERE first_name LIKE """?b*"""
 | KEEP first_name, last_name
 ```
diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md
new file mode 100644
index 0000000000000..3c06662982bbf
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/match.md
@@ -0,0 +1,14 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MATCH
+Performs a match query on the specified field. Returns true if the provided query matches the row.
+
+```
+from books 
+| where match(author, "Faulkner")
+| keep book_no, author 
+| sort book_no 
+| limit 5;
+```
diff --git a/docs/reference/esql/functions/kibana/docs/rlike.md b/docs/reference/esql/functions/kibana/docs/rlike.md
index ed94553e7e44f..95b57799ffe29 100644
--- a/docs/reference/esql/functions/kibana/docs/rlike.md
+++ b/docs/reference/esql/functions/kibana/docs/rlike.md
@@ -10,6 +10,6 @@ expression. The right-hand side of the operator represents the pattern.
 
 ```
 FROM employees
-| WHERE first_name RLIKE ".leja.*"
+| WHERE first_name RLIKE """.leja.*"""
 | KEEP first_name, last_name
 ```
diff --git a/docs/reference/esql/functions/layout/hypot.asciidoc b/docs/reference/esql/functions/layout/hypot.asciidoc
new file mode 100644
index 0000000000000..84376a9f15908
--- /dev/null
+++ b/docs/reference/esql/functions/layout/hypot.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-hypot]]
+=== `HYPOT`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/hypot.svg[Embedded,opts=inline]
+
+include::../parameters/hypot.asciidoc[]
+include::../description/hypot.asciidoc[]
+include::../types/hypot.asciidoc[]
+include::../examples/hypot.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/match.asciidoc b/docs/reference/esql/functions/layout/match.asciidoc
new file mode 100644
index 0000000000000..e62c81548c2b1
--- /dev/null
+++ b/docs/reference/esql/functions/layout/match.asciidoc
@@ -0,0 +1,17 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-match]]
+=== `MATCH`
+
+preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/match.svg[Embedded,opts=inline]
+
+include::../parameters/match.asciidoc[]
+include::../description/match.asciidoc[]
+include::../types/match.asciidoc[]
+include::../examples/match.asciidoc[]
diff --git a/docs/reference/esql/functions/like.asciidoc b/docs/reference/esql/functions/like.asciidoc
index 2298617be5699..a569896bc3c1e 100644
--- a/docs/reference/esql/functions/like.asciidoc
+++ b/docs/reference/esql/functions/like.asciidoc
@@ -23,4 +23,20 @@ include::{esql-specs}/docs.csv-spec[tag=like]
 |===
 include::{esql-specs}/docs.csv-spec[tag=like-result]
 |===
+
+Matching the exact characters `*` and `.` will require escaping.
+The escape character is backslash `\`. Since also backslash is a special character in string literals,
+it will require further escaping.
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/string.csv-spec[tag=likeEscapingSingleQuotes]
+----
+
+To reduce the overhead of escaping, we suggest using triple quotes strings `"""`
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/string.csv-spec[tag=likeEscapingTripleQuotes]
+----
 // end::body[]
diff --git a/docs/reference/esql/functions/math-functions.asciidoc b/docs/reference/esql/functions/math-functions.asciidoc
index e311208795533..9fedfa57f50c5 100644
--- a/docs/reference/esql/functions/math-functions.asciidoc
+++ b/docs/reference/esql/functions/math-functions.asciidoc
@@ -20,6 +20,7 @@
 * <<esql-e>>
 * <<esql-exp>>
 * <<esql-floor>>
+* <<esql-hypot>>
 * <<esql-log>>
 * <<esql-log10>>
 * <<esql-pi>>
@@ -46,6 +47,7 @@ include::layout/cosh.asciidoc[]
 include::layout/e.asciidoc[]
 include::layout/exp.asciidoc[]
 include::layout/floor.asciidoc[]
+include::layout/hypot.asciidoc[]
 include::layout/log.asciidoc[]
 include::layout/log10.asciidoc[]
 include::layout/pi.asciidoc[]
diff --git a/docs/reference/esql/functions/parameters/hypot.asciidoc b/docs/reference/esql/functions/parameters/hypot.asciidoc
new file mode 100644
index 0000000000000..9d6c7d50c7bec
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/hypot.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`number1`::
+Numeric expression. If `null`, the function returns `null`.
+
+`number2`::
+Numeric expression. If `null`, the function returns `null`.
diff --git a/docs/reference/esql/functions/parameters/match.asciidoc b/docs/reference/esql/functions/parameters/match.asciidoc
new file mode 100644
index 0000000000000..f18adb28cd20c
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/match.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`field`::
+Field that the query will target.
+
+`query`::
+Text you wish to find in the provided field.
diff --git a/docs/reference/esql/functions/rlike.asciidoc b/docs/reference/esql/functions/rlike.asciidoc
index 031594ae403da..f6009b2c49528 100644
--- a/docs/reference/esql/functions/rlike.asciidoc
+++ b/docs/reference/esql/functions/rlike.asciidoc
@@ -18,4 +18,20 @@ include::{esql-specs}/docs.csv-spec[tag=rlike]
 |===
 include::{esql-specs}/docs.csv-spec[tag=rlike-result]
 |===
+
+Matching special characters (eg. `.`, `*`, `(`...) will require escaping.
+The escape character is backslash `\`. Since also backslash is a special character in string literals,
+it will require further escaping.
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/string.csv-spec[tag=rlikeEscapingSingleQuotes]
+----
+
+To reduce the overhead of escaping, we suggest using triple quotes strings `"""`
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/string.csv-spec[tag=rlikeEscapingTripleQuotes]
+----
 // end::body[]
diff --git a/docs/reference/esql/functions/signature/hypot.svg b/docs/reference/esql/functions/signature/hypot.svg
new file mode 100644
index 0000000000000..b849ea42cfd9e
--- /dev/null
+++ b/docs/reference/esql/functions/signature/hypot.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="444" height="46" viewbox="0 0 444 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m80 0h10m32 0h10m104 0h10m32 0h10m104 0h10m32 0h5"/><rect class="s" x="5" y="5" width="80" height="36"/><text class="k" x="15" y="31">HYPOT</text><rect class="s" x="95" y="5" width="32" height="36" rx="7"/><text class="syn" x="105" y="31">(</text><rect class="s" x="137" y="5" width="104" height="36" rx="7"/><text class="k" x="147" y="31">number1</text><rect class="s" x="251" y="5" width="32" height="36" rx="7"/><text class="syn" x="261" y="31">,</text><rect class="s" x="293" y="5" width="104" height="36" rx="7"/><text class="k" x="303" y="31">number2</text><rect class="s" x="407" y="5" width="32" height="36" rx="7"/><text class="syn" x="417" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/match.svg b/docs/reference/esql/functions/signature/match.svg
new file mode 100644
index 0000000000000..e7bb001247a9d
--- /dev/null
+++ b/docs/reference/esql/functions/signature/match.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="396" height="46" viewbox="0 0 396 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m80 0h10m32 0h10m80 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="80" height="36"/><text class="k" x="15" y="31">MATCH</text><rect class="s" x="95" y="5" width="32" height="36" rx="7"/><text class="syn" x="105" y="31">(</text><rect class="s" x="137" y="5" width="80" height="36" rx="7"/><text class="k" x="147" y="31">field</text><rect class="s" x="227" y="5" width="32" height="36" rx="7"/><text class="syn" x="237" y="31">,</text><rect class="s" x="269" y="5" width="80" height="36" rx="7"/><text class="k" x="279" y="31">query</text><rect class="s" x="359" y="5" width="32" height="36" rx="7"/><text class="syn" x="369" y="31">)</text></svg>
diff --git a/docs/reference/esql/functions/types/hypot.asciidoc b/docs/reference/esql/functions/types/hypot.asciidoc
new file mode 100644
index 0000000000000..dd06ba96d7f34
--- /dev/null
+++ b/docs/reference/esql/functions/types/hypot.asciidoc
@@ -0,0 +1,24 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+number1 | number2 | result
+double | double | double
+double | integer | double
+double | long | double
+double | unsigned_long | double
+integer | double | double
+integer | integer | double
+integer | long | double
+integer | unsigned_long | double
+long | double | double
+long | integer | double
+long | long | double
+long | unsigned_long | double
+unsigned_long | double | double
+unsigned_long | integer | double
+unsigned_long | long | double
+unsigned_long | unsigned_long | double
+|===
diff --git a/docs/reference/esql/functions/types/match.asciidoc b/docs/reference/esql/functions/types/match.asciidoc
new file mode 100644
index 0000000000000..7523b29c62b1d
--- /dev/null
+++ b/docs/reference/esql/functions/types/match.asciidoc
@@ -0,0 +1,12 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+field | query | result
+keyword | keyword | boolean
+keyword | text | boolean
+text | keyword | boolean
+text | text | boolean
+|===
diff --git a/docs/reference/esql/processing-commands/where.asciidoc b/docs/reference/esql/processing-commands/where.asciidoc
index 407df30c57215..1d6fc1e90d595 100644
--- a/docs/reference/esql/processing-commands/where.asciidoc
+++ b/docs/reference/esql/processing-commands/where.asciidoc
@@ -5,6 +5,13 @@
 The `WHERE` processing command produces a table that contains all the rows from
 the input table for which the provided condition evaluates to `true`.
 
+[TIP]
+====
+In case of value exclusions, fields with `null` values will be excluded from search results. 
+In this context a `null` means either there is an explicit `null` value in the document or there is no value at all.
+For example: `WHERE field != "value"` will be interpreted as `WHERE field != "value" AND field IS NOT NULL`.
+====
+
 **Syntax**
 
 [source,esql]
diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc
index 18882380ce160..1d9c988f7b6c9 100644
--- a/docs/reference/how-to/knn-search.asciidoc
+++ b/docs/reference/how-to/knn-search.asciidoc
@@ -59,8 +59,7 @@ since it relies on separate data structures to perform the search. Before
 using the <<include-exclude, `excludes`>> parameter, make sure to review the
 downsides of omitting fields from `_source`.
 
-Another option is to use  <<synthetic-source,synthetic `_source`>> if all
-your index fields support it.
+Another option is to use  <<synthetic-source,synthetic `_source`>>.
 
 [discrete]
 === Ensure data nodes have enough memory
diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc
index 5f67014d5bb4a..86f195d030223 100644
--- a/docs/reference/how-to/size-your-shards.asciidoc
+++ b/docs/reference/how-to/size-your-shards.asciidoc
@@ -208,6 +208,7 @@ index can be <<indices-delete-index,removed>>. You may then consider setting
 <<indices-add-alias,Create Alias>> against the destination index for the source 
 index's name to point to it for continuity. 
 
+See this https://www.youtube.com/watch?v=sHyNYnwbYro[fixing shard sizes video] for an example troubleshooting walkthrough.
 
 [discrete]
 [[shard-count-recommendation]]
@@ -571,6 +572,8 @@ PUT _cluster/settings
 }
 ----
 
+See this https://www.youtube.com/watch?v=tZKbDegt4-M[fixing "max shards open" video] for an example troubleshooting walkthrough. For more information, see <<troubleshooting-shards-capacity-issues,Troubleshooting shards capacity>>.
+
 [discrete]
 [[troubleshooting-max-docs-limit]]
 ==== Number of documents in the shard cannot exceed [2147483519]
diff --git a/docs/reference/ilm/error-handling.asciidoc b/docs/reference/ilm/error-handling.asciidoc
index f810afc6c2b5f..e8df44653e9c5 100644
--- a/docs/reference/ilm/error-handling.asciidoc
+++ b/docs/reference/ilm/error-handling.asciidoc
@@ -8,6 +8,9 @@ When this happens, {ilm-init} moves the index to an `ERROR` step.
 If {ilm-init} cannot resolve the error automatically, execution is halted  
 until you resolve the underlying issues with the policy, index, or cluster.
 
+See this https://www.youtube.com/watch?v=VCIqkji3IwY[{ilm-init} health video] 
+for example troubleshooting walkthrough.
+
 For example, you might have a `shrink-index` policy that shrinks an index to four shards once it
 is at least five days old: 
 
@@ -183,6 +186,8 @@ The rollover action then manages setting and updating the alias to
 
 Do not explicitly configure this same alias in the aliases section of an index template.
 
+See this https://www.youtube.com/watch?v=Ww5POq4zZtY[resolving `duplicate alias` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index.lifecycle.rollover_alias [x] does not point to index [y]
 
@@ -191,6 +196,8 @@ Either the index is using the wrong alias or the alias does not exist.
 Check the `index.lifecycle.rollover_alias` <<indices-get-settings, index setting>>.
 To see what aliases are configured, use <<cat-alias, _cat/aliases>>.
 
+See this https://www.youtube.com/watch?v=NKSe67x7aw8[resolving `not point to index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Setting [index.lifecycle.rollover_alias] for index [y] is empty or not defined
 
@@ -198,6 +205,8 @@ The `index.lifecycle.rollover_alias` setting must be configured for the rollover
 
 Update the index settings to set `index.lifecycle.rollover_alias`.
 
+See this https://www.youtube.com/watch?v=LRpMC2GS_FQ[resolving `empty or not defined` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Alias [x] has more than one write index [y,z]
 
@@ -205,6 +214,8 @@ Only one index can be designated as the write index for a particular alias.
 
 Use the <<indices-aliases, aliases>> API to set `is_write_index:false` for all but one index.
 
+See this https://www.youtube.com/watch?v=jCUvZCT5Hm4[resolving `more than one write index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index name [x] does not match pattern ^.*-\d+
 
@@ -214,6 +225,8 @@ For example, `my-index` does not match the pattern requirement.
 
 Append a numeric value to the index name, for example `my-index-000001`.
 
+See this https://www.youtube.com/watch?v=9sp1zF6iL00[resolving `does not match pattern` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== CircuitBreakingException: [x] data too large, data for [y]
 
@@ -227,8 +240,7 @@ For more information, see <<circuit-breaker-errors>>.
 
 This indicates that the cluster is running out of disk space.
 This can happen when you don't have {ilm} set up to roll over from hot to warm nodes.
-
-Consider adding nodes, upgrading your hardware, or deleting unneeded indices.
+For more information, see <<fix-watermark-errors,Fix watermark errors>>.
 
 [discrete]
 ==== security_exception: action [<action-name>] is unauthorized for user [<user-name>] with roles [<role-name>], this action is granted by the index privileges [manage_follow_index,manage,all]
diff --git a/docs/reference/images/semantic-options.svg b/docs/reference/images/semantic-options.svg
new file mode 100644
index 0000000000000..3bedf5307357e
--- /dev/null
+++ b/docs/reference/images/semantic-options.svg
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg viewBox="0 0 800 500" xmlns="http://www.w3.org/2000/svg">
+  <!-- Title -->
+  <text x="400" y="40" text-anchor="middle" font-size="24" fill="#000" font-family="Inter, Arial, sans-serif">Elasticsearch semantic search workflows</text>
+  
+  <!-- Workflow boxes -->
+  <g transform="translate(0, 80)">
+    <!-- semantic_text workflow (recommended) -->
+    <rect x="50" y="50" width="200" height="300" rx="10" fill="#c2e0ff" stroke="#3f8dd6" stroke-width="2"/>
+    <text x="150" y="80" text-anchor="middle" font-size="16" font-weight="bold" fill="#2c5282" font-family="Inter, Arial, sans-serif">semantic_text</text>
+    <text x="150" y="100" text-anchor="middle" font-size="12" fill="#2c5282" font-family="Inter, Arial, sans-serif">(Recommended)</text>
+    
+    <!-- Inference API workflow -->
+    <rect x="300" y="50" width="200" height="300" rx="10" fill="#d9f1e3" stroke="#38a169" stroke-width="2"/>
+    <text x="400" y="80" text-anchor="middle" font-size="16" font-weight="bold" fill="#276749" font-family="Inter, Arial, sans-serif">Inference API</text>
+    
+    <!-- Model deployment workflow -->
+    <rect x="550" y="50" width="200" height="300" rx="10" fill="#feebc8" stroke="#dd6b20" stroke-width="2"/>
+    <text x="650" y="80" text-anchor="middle" font-size="16" font-weight="bold" fill="#9c4221" font-family="Inter, Arial, sans-serif">Model Deployment</text>
+    
+    <!-- Complexity indicators -->
+    <text x="150" y="130" text-anchor="middle" font-size="12" fill="#2c5282" font-family="Inter, Arial, sans-serif">Complexity: Low</text>
+    <text x="400" y="130" text-anchor="middle" font-size="12" fill="#276749" font-family="Inter, Arial, sans-serif">Complexity: Medium</text>
+    <text x="650" y="130" text-anchor="middle" font-size="12" fill="#9c4221" font-family="Inter, Arial, sans-serif">Complexity: High</text>
+    
+    <!-- Components in each workflow -->
+    <g transform="translate(60, 150)">
+      <!-- semantic_text components -->
+      <rect x="10" y="0" width="170" height="30" rx="5" fill="#fff" stroke="#3f8dd6"/>
+      <text x="95" y="20" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Create Inference Endpoint</text>
+      
+      <rect x="10" y="40" width="170" height="30" rx="5" fill="#fff" stroke="#3f8dd6"/>
+      <text x="95" y="60" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Define Index Mapping</text>
+      
+      <!-- Inference API components -->
+      <rect x="260" y="0" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
+      <text x="345" y="20" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Create Inference Endpoint</text>
+      
+      <rect x="260" y="40" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
+      <text x="345" y="60" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Configure Model Settings</text>
+      
+      <rect x="260" y="80" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
+      <text x="345" y="100" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Define Index Mapping</text>
+      
+      <rect x="260" y="120" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
+      <text x="345" y="140" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Setup Ingest Pipeline</text>
+      
+      <!-- Model deployment components -->
+      <rect x="510" y="0" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
+      <text x="595" y="20" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Select NLP Model</text>
+      
+      <rect x="510" y="40" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
+      <text x="595" y="60" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Deploy with Eland Client</text>
+      
+      <rect x="510" y="80" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
+      <text x="595" y="100" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Define Index Mapping</text>
+      
+      <rect x="510" y="120" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
+      <text x="595" y="140" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Setup Ingest Pipeline</text>
+    </g>
+  </g>
+</svg>
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index ed8cf6c1494e4..1c8f1db216b75 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -122,7 +122,7 @@ preview:[]
 
     The number of shards a custom <<mapping-routing-field,routing>> value can go to.
     Defaults to 1 and can only be set at index creation time. This value must be less
-    than the `index.number_of_shards` unless the `index.number_of_shards` value is also 1.
+    than the `index.number_of_routing_shards` unless the `index.number_of_routing_shards` value is also 1.
     See <<routing-index-partition>> for more details about how this setting is used.
 
 [[ccr-index-soft-deletes]]
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 7e207146e38e3..9e775afa8f6d1 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -6,7 +6,7 @@ include::links.asciidoc[]
 
 include::landing-page.asciidoc[]
 
-include::release-notes/highlights.asciidoc[]
+// overview / install
 
 include::intro.asciidoc[]
 
@@ -14,33 +14,37 @@ include::quickstart/index.asciidoc[]
 
 include::setup.asciidoc[]
 
-include::upgrade.asciidoc[]
+// search solution
 
-include::index-modules.asciidoc[]
+include::search/search-your-data/search-your-data.asciidoc[]
 
-include::mapping.asciidoc[]
+include::reranking/index.asciidoc[]
 
-include::analysis.asciidoc[]
+// data management
+
+include::index-modules.asciidoc[]
 
 include::indices/index-templates.asciidoc[]
 
-include::data-streams/data-streams.asciidoc[]
+include::alias.asciidoc[]
 
-include::ingest.asciidoc[]
+include::mapping.asciidoc[]
 
-include::alias.asciidoc[]
+include::analysis.asciidoc[]
 
-include::search/search-your-data/search-your-data.asciidoc[]
+include::ingest.asciidoc[]
 
-include::reranking/index.asciidoc[]
+include::connector/docs/index.asciidoc[]
 
-include::query-dsl.asciidoc[]
+include::data-streams/data-streams.asciidoc[]
 
-include::aggregations.asciidoc[]
+include::data-management.asciidoc[]
 
-include::geospatial-analysis.asciidoc[]
+include::data-rollup-transform.asciidoc[]
 
-include::connector/docs/index.asciidoc[]
+// analysis tools
+
+include::query-dsl.asciidoc[]
 
 include::eql/eql.asciidoc[]
 
@@ -50,34 +54,50 @@ include::sql/index.asciidoc[]
 
 include::scripting.asciidoc[]
 
-include::data-management.asciidoc[]
+include::aggregations.asciidoc[]
 
-include::autoscaling/index.asciidoc[]
+include::geospatial-analysis.asciidoc[]
+
+include::watcher/index.asciidoc[]
+
+// cluster management
 
 include::monitoring/index.asciidoc[]
 
-include::data-rollup-transform.asciidoc[]
+include::security/index.asciidoc[]
+
+// production tasks
 
 include::high-availability.asciidoc[]
 
+include::how-to.asciidoc[]
+
+include::autoscaling/index.asciidoc[]
+
 include::snapshot-restore/index.asciidoc[]
 
-include::security/index.asciidoc[]
+// reference
 
-include::watcher/index.asciidoc[]
+include::reference-architectures.asciidoc[]
 
-include::commands/index.asciidoc[]
+include::rest-api/index.asciidoc[]
 
-include::how-to.asciidoc[]
+include::commands/index.asciidoc[]
 
 include::troubleshooting.asciidoc[]
 
-include::rest-api/index.asciidoc[]
+// upgrades
+
+include::upgrade.asciidoc[]
 
 include::migration/index.asciidoc[]
 
+include::release-notes/highlights.asciidoc[]
+
 include::release-notes.asciidoc[]
 
 include::dependencies-versions.asciidoc[]
 
+// etc
+
 include::redirects.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/indices/alias-exists.asciidoc b/docs/reference/indices/alias-exists.asciidoc
index f820a95028a0f..d7b3454dcff56 100644
--- a/docs/reference/indices/alias-exists.asciidoc
+++ b/docs/reference/indices/alias-exists.asciidoc
@@ -52,8 +52,6 @@ Defaults to `all`.
 (Optional, Boolean) If `false`, requests that include a missing data stream or
 index in the `<target>` return an error. Defaults to `false`.
 
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]
-
 [[alias-exists-api-response-codes]]
 ==== {api-response-codes-title}
 
diff --git a/docs/reference/indices/get-alias.asciidoc b/docs/reference/indices/get-alias.asciidoc
index 743aaf7aee174..41d62fb70e01b 100644
--- a/docs/reference/indices/get-alias.asciidoc
+++ b/docs/reference/indices/get-alias.asciidoc
@@ -58,5 +58,3 @@ Defaults to `all`.
 `ignore_unavailable`::
 (Optional, Boolean) If `false`, requests that include a missing data stream or
 index in the `<target>` return an error. Defaults to `false`.
-
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]
diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc
index bee39bf9b9851..a83fb1a516b80 100644
--- a/docs/reference/inference/delete-inference.asciidoc
+++ b/docs/reference/inference/delete-inference.asciidoc
@@ -2,16 +2,11 @@
 [[delete-inference-api]]
 === Delete {infer} API
 
-experimental[]
-
 Deletes an {infer} endpoint.
 
-IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or
-Hugging Face. For built-in models and models uploaded through Eland, the {infer}
-APIs offer an alternative way to use and manage trained models. However, if you
-do not plan to use the {infer} APIs to use these models or if you want to use
-non-NLP models, use the <<ml-df-trained-models-apis>>.
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
 
 
 [discrete]
diff --git a/docs/reference/inference/get-inference.asciidoc b/docs/reference/inference/get-inference.asciidoc
index c3fe841603bcc..16e38d2aa148b 100644
--- a/docs/reference/inference/get-inference.asciidoc
+++ b/docs/reference/inference/get-inference.asciidoc
@@ -2,16 +2,11 @@
 [[get-inference-api]]
 === Get {infer} API
 
-experimental[]
-
 Retrieves {infer} endpoint information.
 
-IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or
-Hugging Face. For built-in models and models uploaded through Eland, the {infer}
-APIs offer an alternative way to use and manage trained models. However, if you
-do not plan to use the {infer} APIs to use these models or if you want to use
-non-NLP models, use the <<ml-df-trained-models-apis>>.
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
 
 
 [discrete]
diff --git a/docs/reference/inference/images/inference-landscape.jpg b/docs/reference/inference/images/inference-landscape.jpg
new file mode 100644
index 0000000000000..d66d67763cab5
Binary files /dev/null and b/docs/reference/inference/images/inference-landscape.jpg differ
diff --git a/docs/reference/inference/images/inference-landscape.png b/docs/reference/inference/images/inference-landscape.png
deleted file mode 100644
index a35d1370fd09b..0000000000000
Binary files a/docs/reference/inference/images/inference-landscape.png and /dev/null differ
diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc
index 8fdf8aecc2ae5..ddcff1abc7dce 100644
--- a/docs/reference/inference/inference-apis.asciidoc
+++ b/docs/reference/inference/inference-apis.asciidoc
@@ -2,8 +2,6 @@
 [[inference-apis]]
 == {infer-cap} APIs
 
-experimental[]
-
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
 {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure,
 Google AI Studio or Hugging Face. For built-in models and models uploaded
@@ -21,10 +19,11 @@ the following APIs to manage {infer} models and perform {infer}:
 * <<get-inference-api>>
 * <<post-inference-api>>
 * <<put-inference-api>>
+* <<update-inference-api>>
 
 [[inference-landscape]]
 .A representation of the Elastic inference landscape
-image::images/inference-landscape.png[A representation of the Elastic inference landscape,align="center"]
+image::images/inference-landscape.jpg[A representation of the Elastic inference landscape,align="center"]
 
 An {infer} endpoint enables you to use the corresponding {ml} model without
 manual deployment and apply it to your data at ingestion time through
@@ -35,10 +34,29 @@ Elastic –, then create an {infer} endpoint by the <<put-inference-api>>.
 Now use <<semantic-search-semantic-text, semantic text>> to perform
 <<semantic-search, semantic search>> on your data.
 
+
+[discrete]
+[[default-enpoints]]
+=== Default {infer} endpoints
+
+Your {es} deployment contains some preconfigured {infer} endpoints that makes it easier for you to use them when defining `semantic_text` fields or {infer} processors.
+The following list contains the default {infer} endpoints listed by `inference_id`:
+
+* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts)
+* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts)
+
+Use the `inference_id` of the endpoint in a <<semantic-text,`semantic_text`>> field definition or when creating an <<inference-processor,{infer} processor>>.
+The API call will automatically download and deploy the model which might take a couple of minutes.
+Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled.
+For these models, the minimum number of allocations is `0`. 
+If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes.
+
+
 include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
 include::put-inference.asciidoc[]
+include::update-inference.asciidoc[]
 include::service-alibabacloud-ai-search.asciidoc[]
 include::service-amazon-bedrock.asciidoc[]
 include::service-anthropic.asciidoc[]
@@ -52,3 +70,4 @@ include::service-google-vertex-ai.asciidoc[]
 include::service-hugging-face.asciidoc[]
 include::service-mistral.asciidoc[]
 include::service-openai.asciidoc[]
+include::service-watsonx-ai.asciidoc[]
diff --git a/docs/reference/inference/post-inference.asciidoc b/docs/reference/inference/post-inference.asciidoc
index 52131c0b10776..4edefcc911e2e 100644
--- a/docs/reference/inference/post-inference.asciidoc
+++ b/docs/reference/inference/post-inference.asciidoc
@@ -2,16 +2,11 @@
 [[post-inference-api]]
 === Perform inference API
 
-experimental[]
-
 Performs an inference task on an input text by using an {infer} endpoint.
 
-IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or
-Hugging Face. For built-in models and models uploaded through Eland, the {infer}
-APIs offer an alternative way to use and manage trained models. However, if you
-do not plan to use the {infer} APIs to use these models or if you want to use
-non-NLP models, use the <<ml-df-trained-models-apis>>.
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
 
 
 [discrete]
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index 96e127e741d56..e7e25ec98b49d 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -2,19 +2,12 @@
 [[put-inference-api]]
 === Create {infer} API
 
-experimental[]
-
 Creates an {infer} endpoint to perform an {infer} task.
 
 [IMPORTANT]
 ====
-* The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral,
-Azure OpenAI, Google AI Studio, Google Vertex AI, Anthropic or Hugging Face.
-* For built-in models and models uploaded through Eland, the {infer} APIs offer an
-alternative way to use and manage trained models. However, if you do not plan to
-use the {infer} APIs to use these models or if you want to use non-NLP models,
-use the <<ml-df-trained-models-apis>>.
+* The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral, Azure OpenAI, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
+* For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
 ====
 
 
@@ -71,6 +64,7 @@ Click the links to review the configuration details of the services:
 * <<infer-service-hugging-face,Hugging Face>> (`text_embedding`)
 * <<infer-service-mistral,Mistral>> (`text_embedding`)
 * <<infer-service-openai,OpenAI>> (`completion`, `text_embedding`)
+* <<infer-service-watsonx-ai>> (`text_embedding`)
 
 The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of
 the services connect to external providers.
\ No newline at end of file
diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc
index efa0c78b8356f..259779a12134d 100644
--- a/docs/reference/inference/service-elasticsearch.asciidoc
+++ b/docs/reference/inference/service-elasticsearch.asciidoc
@@ -1,12 +1,9 @@
 [[infer-service-elasticsearch]]
 === Elasticsearch {infer} service
 
-Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch`
-service.
+Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` service.
 
-NOTE: If you use the E5 model through the `elasticsearch` service, the API
-request will automatically download and deploy the model if it isn't downloaded
-yet.
+NOTE: If you use the ELSER or the E5 model through the `elasticsearch` service, the API request will automatically download and deploy the model if it isn't downloaded yet.
 
 
 [discrete]
@@ -56,6 +53,11 @@ These settings are specific to the `elasticsearch` service.
 (Optional, object)
 include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
 
+`deployment_id`:::
+(Optional, string)
+The `deployment_id` of an existing trained model deployment.
+When `deployment_id` is used the `model_id` is optional.
+
 `enabled`::::
 (Optional, Boolean)
 include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-enabled]
@@ -71,7 +73,7 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-min-number]
 `model_id`:::
 (Required, string)
 The name of the model to use for the {infer} task.
-It can be the ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or a text embedding model already
+It can be the ID of either a built-in model (for example, `.multilingual-e5-small` for E5), a text embedding model already
 {ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
 
 `num_allocations`:::
@@ -98,15 +100,44 @@ Returns the document instead of only the index. Defaults to `true`.
 =====
 
 
+[discrete]
+[[inference-example-elasticsearch-elser]]
+==== ELSER via the `elasticsearch` service
+
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type.
+
+The API request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/my-elser-model
+{
+  "service": "elasticsearch",
+  "service_settings": {
+    "adaptive_allocations": { <1>
+      "enabled": true,
+      "min_number_of_allocations": 1,
+      "max_number_of_allocations": 10
+    },
+    "num_threads": 1,
+    "model_id": ".elser_model_2" <2>
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> Adaptive allocations will be enabled with the minimum of 1 and the maximum of 10 allocations.
+<2> The `model_id` must be the ID of one of the built-in ELSER models.
+Valid values are `.elser_model_2` and `.elser_model_2_linux-x86_64`.
+For further details, refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation].
+
+
 [discrete]
 [[inference-example-elasticsearch]]
 ==== E5 via the `elasticsearch` service
 
-The following example shows how to create an {infer} endpoint called
-`my-e5-model` to perform a `text_embedding` task type.
+The following example shows how to create an {infer} endpoint called `my-e5-model` to perform a `text_embedding` task type.
 
-The API request below will automatically download the E5 model if it isn't
-already downloaded and then deploy the model.
+The API request below will automatically download the E5 model if it isn't already downloaded and then deploy the model.
 
 [source,console]
 ------------------------------------------------------------
@@ -185,3 +216,46 @@ PUT _inference/text_embedding/my-e5-model
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
+
+
+[discrete]
+[[inference-example-existing-deployment]]
+==== Using an existing model deployment with the `elasticsearch` service
+
+The following example shows how to use an already existing model deployment when creating an {infer} endpoint.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/use_existing_deployment
+{
+  "service": "elasticsearch",
+  "service_settings": {
+    "deployment_id": ".elser_model_2" <1>
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The `deployment_id` of the already existing model deployment.
+
+The API response contains the `model_id`, and the threads and allocations settings from the model deployment:
+
+[source,console-result]
+------------------------------------------------------------
+{
+  "inference_id": "use_existing_deployment",
+  "task_type": "sparse_embedding",
+  "service": "elasticsearch",
+  "service_settings": {
+    "num_allocations": 2,
+    "num_threads": 1,
+    "model_id": ".elser_model_2",
+    "deployment_id": ".elser_model_2"
+  },
+  "chunking_settings": {
+    "strategy": "sentence",
+    "max_chunk_size": 250,
+    "sentence_overlap": 1
+  }
+}
+------------------------------------------------------------
+// NOTCONSOLE
\ No newline at end of file
diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc
index c7217f38d459b..521fab0375584 100644
--- a/docs/reference/inference/service-elser.asciidoc
+++ b/docs/reference/inference/service-elser.asciidoc
@@ -2,6 +2,7 @@
 === ELSER {infer} service
 
 Creates an {infer} endpoint to perform an {infer} task with the `elser` service.
+You can also deploy ELSER by using the <<infer-service-elasticsearch>>.
 
 NOTE: The API request will automatically download and deploy the ELSER model if
 it isn't already downloaded.
@@ -80,12 +81,13 @@ Must be a power of 2. Max allowed value is 32.
 [[inference-example-elser]]
 ==== ELSER service example
 
-The following example shows how to create an {infer} endpoint called
-`my-elser-model` to perform a `sparse_embedding` task type.
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type.
 Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info.
 
-The request below will automatically download the ELSER model if it isn't
-already downloaded and then deploy the model.
+NOTE: If you want to optimize your ELSER endpoint for ingest, set the number of threads to `1` (`"num_threads": 1`).
+If you want to optimize your ELSER endpoint for search, set the number of threads to greater than `1`.
+
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
 
 [source,console]
 ------------------------------------------------------------
@@ -100,7 +102,6 @@ PUT _inference/sparse_embedding/my-elser-model
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-
 Example response:
 
 [source,console-result]
@@ -128,14 +129,14 @@ If using the Python client, you can set the `timeout` parameter to a higher valu
 
 [discrete]
 [[inference-example-elser-adaptive-allocation]]
-==== Setting adaptive allocation for the ELSER service
+==== Setting adaptive allocations for the ELSER service
+
+NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
+To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
 
-The following example shows how to create an {infer} endpoint called
-`my-elser-model` to perform a `sparse_embedding` task type and configure
-adaptive allocations.
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations.
 
-The request below will automatically download the ELSER model if it isn't
-already downloaded and then deploy the model.
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
 
 [source,console]
 ------------------------------------------------------------
diff --git a/docs/reference/inference/service-watsonx-ai.asciidoc b/docs/reference/inference/service-watsonx-ai.asciidoc
new file mode 100644
index 0000000000000..597afc27fd0cf
--- /dev/null
+++ b/docs/reference/inference/service-watsonx-ai.asciidoc
@@ -0,0 +1,115 @@
+[[infer-service-watsonx-ai]]
+=== Watsonx {infer} service
+
+Creates an {infer} endpoint to perform an {infer} task with the `watsonxai` service.
+
+You need an https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-provisioning&interface=api[IBM Cloud® Databases for Elasticsearch deployment] to use the `watsonxai` {infer} service.
+You can provision one through the https://cloud.ibm.com/databases/databases-for-elasticsearch/create[IBM catalog], the https://cloud.ibm.com/docs/databases-cli-plugin?topic=databases-cli-plugin-cdb-reference[Cloud Databases CLI plug-in], the https://cloud.ibm.com/apidocs/cloud-databases-api[Cloud Databases API], or https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database[Terraform].
+
+
+[discrete]
+[[infer-service-watsonx-ai-api-request]]
+==== {api-request-title}
+
+`PUT /_inference/<task_type>/<inference_id>`
+
+[discrete]
+[[infer-service-watsonx-ai-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=inference-id]
+
+`<task_type>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=task-type]
++
+--
+Available task types:
+
+* `text_embedding`.
+--
+
+[discrete]
+[[infer-service-watsonx-ai-api-request-body]]
+==== {api-request-body-title}
+
+`service`::
+(Required, string)
+The type of service supported for the specified task type. In this case, 
+`watsonxai`.
+
+`service_settings`::
+(Required, object)
+include::inference-shared.asciidoc[tag=service-settings]
++
+--
+These settings are specific to the `watsonxai` service.
+--
+
+`api_key`:::
+(Required, string)
+A valid API key of your Watsonx account.
+You can find your Watsonx API keys or you can create a new one https://cloud.ibm.com/iam/apikeys[on the API keys page].
++
+--
+include::inference-shared.asciidoc[tag=api-key-admonition]
+--
+
+`api_version`:::
+(Required, string)
+Version parameter that takes a version date in the format of `YYYY-MM-DD`.
+For the active version data parameters, refer to the https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[documentation].
+
+`model_id`:::
+(Required, string)
+The name of the model to use for the {infer} task.
+Refer to the IBM Embedding Models section in the https://www.ibm.com/products/watsonx-ai/foundation-models[Watsonx documentation] for the list of available text embedding models.
+
+`url`:::
+(Required, string)
+The URL endpoint to use for the requests.
+
+`project_id`:::
+(Required, string)
+The name of the project to use for the {infer} task.
+
+`rate_limit`:::
+(Optional, object)
+By default, the `watsonxai` service sets the number of requests allowed per minute to `120`.
+This helps to minimize the number of rate limit errors returned from Watsonx.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+--
+include::inference-shared.asciidoc[tag=request-per-minute-example]
+--
+
+
+[discrete]
+[[inference-example-watsonx-ai]]
+==== Watsonx AI service example
+
+The following example shows how to create an {infer} endpoint called `watsonx-embeddings` to perform a `text_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/watsonx-embeddings
+{
+    "service": "watsonxai",
+    "service_settings": {
+        "api_key": "<api_key>", <1>
+        "url": "<url>", <2>
+        "model_id": "ibm/slate-30m-english-rtrvr",
+        "project_id": "<project_id>", <3>
+        "api_version": "2024-03-14" <4>
+    }
+}
+
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> A valid Watsonx API key.
+You can find on the https://cloud.ibm.com/iam/apikeys[API keys page of your account].
+<2> The {infer} endpoint URL you created on Watsonx.
+<3> The ID of your IBM Cloud project.
+<4> A valid API version parameter. You can find the active version data parameters https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[here].
\ No newline at end of file
diff --git a/docs/reference/inference/update-inference.asciidoc b/docs/reference/inference/update-inference.asciidoc
new file mode 100644
index 0000000000000..efd29231ac12e
--- /dev/null
+++ b/docs/reference/inference/update-inference.asciidoc
@@ -0,0 +1,85 @@
+[role="xpack"]
+[[update-inference-api]]
+=== Update inference API
+
+Updates an {infer} endpoint.
+
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
+
+
+[discrete]
+[[update-inference-api-request]]
+==== {api-request-title}
+
+`POST _inference/<inference_id>/_update`
+
+`POST _inference/<task_type>/<inference_id>/_update`
+
+
+[discrete]
+[[update-inference-api-prereqs]]
+==== {api-prereq-title}
+
+* Requires the `manage_inference` <<privileges-list-cluster,cluster privilege>> (the built-in inference_admin role grants this privilege)
+* Requires an existing {infer} endpoint, created by using the <<put-inference-api>>
+
+
+[discrete]
+[[update-inference-api-desc]]
+==== {api-description-title}
+
+The update inference API enables you to update the task_settings, secrets, and/or num_allocations of an existing {infer} endpoint.
+
+To use the update API, you can modify `task_settings`, secrets (within `service_settings`), or `num_allocations`, depending on the specific endpoint service and task_type you've created.
+To view the updatable `task_settings`, the field names of secrets (specific to each service), and the services where `num_allocations` is applicable (only for the `elasticsearch` service), refer to the following list of services available through the {infer} API.
+You will find the available task types next to each service name.
+Click the links to review the service configuration details:
+
+* <<infer-service-alibabacloud-ai-search,AlibabaCloud AI Search>> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`)
+* <<infer-service-amazon-bedrock,Amazon Bedrock>> (`completion`, `text_embedding`)
+* <<infer-service-anthropic,Anthropic>> (`completion`)
+* <<infer-service-azure-ai-studio,Azure AI Studio>> (`completion`, `text_embedding`)
+* <<infer-service-azure-openai,Azure OpenAI>> (`completion`, `text_embedding`)
+* <<infer-service-cohere,Cohere>> (`completion`, `rerank`, `text_embedding`)
+* <<infer-service-elasticsearch,Elasticsearch>> (`rerank`, `sparse_embedding`, `text_embedding` - this service is for built-in models and models uploaded through Eland)
+* <<infer-service-elser,ELSER>> (`sparse_embedding`)
+* <<infer-service-google-ai-studio,Google AI Studio>> (`completion`, `text_embedding`)
+* <<infer-service-google-vertex-ai,Google Vertex AI>> (`rerank`, `text_embedding`) 
+* <<infer-service-hugging-face,Hugging Face>> (`text_embedding`)
+* <<infer-service-mistral,Mistral>> (`text_embedding`)
+* <<infer-service-openai,OpenAI>> (`completion`, `text_embedding`)
+
+
+[discrete]
+[[update-inference-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+The unique identifier of the {infer} endpoint.
+
+
+`<task_type>`::
+(Optional, string)
+The type of {infer} task that the model performs.
+Refer to the service list in the <<put-inference-api-desc,API description section>> for the available task types.
+
+
+[discrete]
+[[update-inference-api-example]]
+==== {api-examples-title}
+
+The following example shows how to update an API key of an {infer} endpoint called `my-inference-endpoint`:
+
+[source,console]
+------------------------------------------------------------
+POST _inference/my-inference-endpoint/_update
+{
+ "service_settings": {
+   "api_key": "<API_KEY>"
+ }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc
index 1bee03ea3e58a..da591eed7546f 100644
--- a/docs/reference/ingest/apis/simulate-ingest.asciidoc
+++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc
@@ -108,6 +108,14 @@ POST /_ingest/_simulate
       "index_patterns": ["my-index-*"],
       "composed_of": ["component_template_1", "component_template_2"]
     }
+  },
+  "mapping_addition": { <4>
+    "dynamic": "strict",
+    "properties": {
+      "foo": {
+        "type": "keyword"
+      }
+    }
   }
 }
 ----
@@ -117,6 +125,7 @@ POST /_ingest/_simulate
 These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
 <3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request.
 These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
+<4> This mapping is merged into the index's final mapping just before validation. It is used only for the duration of this request.
 
 [[simulate-ingest-api-request]]
 ==== {api-request-title}
@@ -246,6 +255,10 @@ include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body]
 
 ====
 
+`mapping_addition`::
+(Optional, <<mapping,mapping object>>)
+Definition of a mapping that will be merged into the index's mapping for validation during the course of this request.
+
 [[simulate-ingest-api-example]]
 ==== {api-examples-title}
 
diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc
index d2bbc1c8643ce..7f1bad1798fc8 100644
--- a/docs/reference/ingest/processors/user-agent.asciidoc
+++ b/docs/reference/ingest/processors/user-agent.asciidoc
@@ -20,7 +20,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab
 | `field`                | yes       | -                                                                                               | The field containing the user agent string.
 | `target_field`         | no        | user_agent                                                                                      | The field that will be filled with the user agent details.
 | `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
-| `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
+| `properties`           | no        | [`name`, `os`, `device`, `original`, `version`] | Controls what properties are added to `target_field`.
 | `extract_device_type`  | no        | `false`                                                                                         | beta:[] Extracts device type from the user agent string on a best-effort basis.
 | `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc
index 902b6c26611e5..f8666e2993d6a 100644
--- a/docs/reference/mapping/fields/synthetic-source.asciidoc
+++ b/docs/reference/mapping/fields/synthetic-source.asciidoc
@@ -2,7 +2,7 @@
 ==== Synthetic `_source`
 
 IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
-(indices that have `index.mode` set to `time_series`). For other indices
+(indices that have `index.mode` set to `time_series`). For other indices,
 synthetic `_source` is in technical preview. Features in technical preview may
 be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
@@ -11,15 +11,19 @@ of official GA features.
 Though very handy to have around, the source field takes up a significant amount
 of space on disk. Instead of storing source documents on disk exactly as you
 send them, Elasticsearch can reconstruct source content on the fly upon retrieval.
-Enable this by setting `mode: synthetic` in `_source`:
+Enable this by using the value `synthetic` for the index setting `index.mapping.source.mode`:
 
 [source,console,id=enable-synthetic-source-example]
 ----
 PUT idx
 {
-  "mappings": {
-    "_source": {
-      "mode": "synthetic"
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
     }
   }
 }
@@ -38,7 +42,7 @@ properties when used with synthetic `_source`.
 <<synthetic-source-fields-native-list, Most field types>> construct synthetic `_source` using existing data, most
 commonly <<doc-values,`doc_values`>> and <<stored-fields, stored fields>>. For these field types, no additional space
 is needed to store the contents of `_source` field. Due to the storage layout of <<doc-values,`doc_values`>>, the
-generated `_source` field undergoes <<synthetic-source-modifications, modifications>> compared to original document.
+generated `_source` field undergoes <<synthetic-source-modifications, modifications>> compared to the original document.
 
 For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in
 non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original
@@ -227,10 +231,16 @@ For instance:
 ----
 PUT idx_keep
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": {
-      "mode": "synthetic"
-    },
     "properties": {
       "path": {
         "type": "object",
diff --git a/docs/reference/mapping/params/format.asciidoc b/docs/reference/mapping/params/format.asciidoc
index 943e8fb879ff3..6c82b04eb5fe5 100644
--- a/docs/reference/mapping/params/format.asciidoc
+++ b/docs/reference/mapping/params/format.asciidoc
@@ -34,13 +34,13 @@ down to the nearest day.
 Completely customizable date formats are supported. The syntax for these is explained in
 https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/time/format/DateTimeFormatter.html[DateTimeFormatter docs].
 
-Note that whilst the built-in formats for week dates use the ISO definition of weekyears,
+Note that while the built-in formats for week dates use the ISO definition of weekyears,
 custom formatters using the `Y`, `W`, or `w` field specifiers use the JDK locale definition
 of weekyears. This can result in different values between the built-in formats and custom formats
 for week dates.
 
 [[built-in-date-formats]]
-==== Built In Formats
+==== Built-in formats
 
 Most of the below formats have a `strict` companion format, which means that
 year, month and day parts of the month must use respectively 4, 2 and 2 digits
diff --git a/docs/reference/mapping/params/subobjects.asciidoc b/docs/reference/mapping/params/subobjects.asciidoc
index b0a5d3817c332..ff91f07cfb359 100644
--- a/docs/reference/mapping/params/subobjects.asciidoc
+++ b/docs/reference/mapping/params/subobjects.asciidoc
@@ -111,6 +111,7 @@ PUT my-index-000001/_doc/metric_1
 
 The `subobjects` setting for existing fields and the top-level mapping definition cannot be updated.
 
+[[subobjects-auto-flattening]]
 ==== Auto-flattening object mappings
 
 It is generally recommended to define the properties of an object that is configured with `subobjects: false` with dotted field names
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index 7e2e7083fa70b..babe4f508b5f0 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -35,12 +35,13 @@ Dates::                 Date types, including <<date,`date`>> and
 [[object-types]]
 ==== Objects and relational types
 
-<<object,`object`>>::         A JSON object.
-<<flattened,`flattened`>>::   An entire JSON object as a single field value.
-<<nested,`nested`>>::         A JSON object that preserves the relationship
-                              between its subfields.
-<<parent-join,`join`>>::      Defines a parent/child relationship for documents
-                              in the same index.
+<<object,`object`>>::           A JSON object.
+<<flattened,`flattened`>>::     An entire JSON object as a single field value.
+<<nested,`nested`>>::           A JSON object that preserves the relationship
+                                between its subfields.
+<<parent-join,`join`>>::        Defines a parent/child relationship for documents
+                                in the same index.
+<<passthrough,`passthrough`>>:: Provides aliases for sub-fields at the same level.
 
 
 [discrete]
@@ -167,6 +168,8 @@ include::types/numeric.asciidoc[]
 
 include::types/object.asciidoc[]
 
+include::types/passthrough.asciidoc[]
+
 include::types/percolator.asciidoc[]
 
 include::types/point.asciidoc[]
diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc
index 8e14fba976360..faae5118e42bb 100644
--- a/docs/reference/mapping/types/aggregate-metric-double.asciidoc
+++ b/docs/reference/mapping/types/aggregate-metric-double.asciidoc
@@ -259,16 +259,21 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`aggregate_metric-double` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration.
-
 For example:
 [source,console,id=synthetic-source-aggregate-metric-double-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "agg_metric": {
         "type": "aggregate_metric_double",
diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc
index a06e5b4f572e0..81ba44c954e0a 100644
--- a/docs/reference/mapping/types/binary.asciidoc
+++ b/docs/reference/mapping/types/binary.asciidoc
@@ -63,13 +63,21 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`binary` fields support <<synthetic-source,synthetic `_source`>> only when <<doc-values,`doc_values`>> are enabled. Synthetic source always sorts `binary` values in order of their byte representation. For example:
+Synthetic source may sort `binary` values in order of their byte representation. For example:
 [source,console,id=synthetic-source-binary-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "binary": { "type": "binary", "doc_values": true }
     }
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index 32f3d13edf581..268be9016987f 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -241,16 +241,23 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `boolean` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
+default configuration.
 
-Synthetic source always sorts `boolean` fields. For example:
+Synthetic source may sort `boolean` field values. For example:
 [source,console,id=synthetic-source-boolean-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "bool": { "type": "boolean" }
     }
diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
index ca2c23f932fc3..4261d502ca104 100644
--- a/docs/reference/mapping/types/date.asciidoc
+++ b/docs/reference/mapping/types/date.asciidoc
@@ -130,7 +130,7 @@ The following parameters are accepted by `date` fields:
 <<ignore-malformed,`ignore_malformed`>>::
 
     If `true`, malformed numbers are ignored. If `false` (default), malformed
-    numbers throw an exception and reject the whole document.  Note that this
+    numbers throw an exception and reject the whole document. Note that this
     cannot be set if the `script` parameter is used.
 
 <<mapping-index,`index`>>::
@@ -239,17 +239,21 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`date` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `date` fields. For example:
+Synthetic source may sort `date` field values. For example:
 [source,console,id=synthetic-source-date-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "date": { "type": "date" }
     }
diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc
index 1a3b390b1690c..31f5ae09e7a63 100644
--- a/docs/reference/mapping/types/date_nanos.asciidoc
+++ b/docs/reference/mapping/types/date_nanos.asciidoc
@@ -150,18 +150,21 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`date_nanos` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>, <<ignore-malformed,`ignore_malformed`>> set to true
-or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `date_nanos` fields. For example:
+Synthetic source may sort `date_nanos` field values. For example:
 [source,console,id=synthetic-source-date-nanos-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "date": { "type": "date_nanos" }
     }
diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
index 0cd9ee0578b70..44f90eded8632 100644
--- a/docs/reference/mapping/types/dense-vector.asciidoc
+++ b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -115,22 +115,27 @@ that sacrifices result accuracy for improved speed.
 ==== Automatically quantize vectors for kNN search
 
 The `dense_vector` type supports quantization to reduce the memory footprint required when <<approximate-knn, searching>> `float` vectors.
-The two following quantization strategies are supported:
+The three following quantization strategies are supported:
 
 +
 --
-`int8` - Quantizes each dimension of the vector to 1-byte integers. This can reduce the memory footprint by 75% at the cost of some accuracy.
-`int4` - Quantizes each dimension of the vector to half-byte integers. This can reduce the memory footprint by 87% at the cost of some accuracy.
+`int8` - Quantizes each dimension of the vector to 1-byte integers. This reduces the memory footprint by 75% (or 4x) at the cost of some accuracy.
+`int4` - Quantizes each dimension of the vector to half-byte integers. This reduces the memory footprint by 87% (or 8x) at the cost of accuracy.
+`bbq` - experimental:[] Better binary quantization which reduces each dimension to a single bit precision. This reduces the memory footprint by 96% (or 32x) at a larger cost of accuracy. Generally, oversampling during query time and reranking can help mitigate the accuracy loss.
 --
 
-To use a quantized index, you can set your index type to `int8_hnsw` or `int4_hnsw`. When indexing `float` vectors, the current default
+When using a quantized format, you may want to oversample and rescore the results to improve accuracy. See <<dense-vector-knn-search-reranking, oversampling and rescoring>> for more information.
+
+To use a quantized index, you can set your index type to `int8_hnsw`, `int4_hnsw`, or `bbq_hnsw`. When indexing `float` vectors, the current default
 index type is `int8_hnsw`.
 
 NOTE: Quantization will continue to keep the raw float vector values on disk for reranking, reindexing, and quantization improvements over the lifetime of the data.
-This means disk usage will increase by ~25% for `int8` and ~12.5% for `int4` due to the overhead of storing the quantized and raw vectors.
+This means disk usage will increase by ~25% for `int8`, ~12.5% for `int4`, and ~3.1% for `bbq` due to the overhead of storing the quantized and raw vectors.
 
 NOTE: `int4` quantization requires an even number of vector dimensions.
 
+NOTE: experimental:[] `bbq` quantization only supports vector dimensions that are greater than 64.
+
 Here is an example of how to create a byte-quantized index:
 
 [source,console]
@@ -173,6 +178,27 @@ PUT my-byte-quantized-index
 }
 --------------------------------------------------
 
+experimental:[] Here is an example of how to create a binary quantized index:
+
+[source,console]
+--------------------------------------------------
+PUT my-byte-quantized-index
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "dense_vector",
+        "dims": 64,
+        "index": true,
+        "index_options": {
+          "type": "bbq_hnsw"
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
 [role="child_attributes"]
 [[dense-vector-params]]
 ==== Parameters for dense vector fields
@@ -301,11 +327,16 @@ by 4x at the cost of some accuracy. See <<dense-vector-quantization, Automatical
 * `int4_hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically scalar
 quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint
 by 8x at the cost of some accuracy. See <<dense-vector-quantization, Automatically quantize vectors for kNN search>>.
+* experimental:[] `bbq_hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically binary
+quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint
+by 32x at the cost of accuracy. See <<dense-vector-quantization, Automatically quantize vectors for kNN search>>.
 * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values.
 * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports
 `element_type` of `float`.
 * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports
 `element_type` of `float`.
+* experimental:[] `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatically binary quantization. Only supports
+`element_type` of `float`.
 --
 `m`:::
 (Optional, integer)
diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc
index 0a72ebc98ecef..96b230794003a 100644
--- a/docs/reference/mapping/types/flattened.asciidoc
+++ b/docs/reference/mapping/types/flattened.asciidoc
@@ -325,17 +325,24 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 Flattened fields support <<synthetic-source,synthetic`_source`>> in their default
-configuration. Synthetic `_source` cannot be used with <<doc-values,`doc_values`>>
-disabled.
+configuration.
 
-Synthetic source always sorts alphabetically and de-duplicates flattened fields.
+Synthetic source may sort `flattened` field values and remove duplicates.
 For example:
 [source,console,id=synthetic-source-flattened-sorting-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "flattened": { "type": "flattened" }
     }
@@ -367,8 +374,16 @@ For example:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "flattened": { "type": "flattened" }
     }
@@ -407,8 +422,16 @@ For example:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "flattened": { "type": "flattened" }
     }
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index 6db05188dfb98..0958997d3fb00 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -219,18 +219,22 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`geo_point` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with <<copy-to,`copy_to`>> or with
-<<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `geo_point` fields (first by latitude and then
+Synthetic source may sort `geo_point` fields (first by latitude and then
 longitude) and reduces them to their stored precision. For example:
 [source,console,id=synthetic-source-geo-point-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "point": { "type": "geo_point" }
     }
diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc
index e50c7d73b1b76..affebc6f721e4 100644
--- a/docs/reference/mapping/types/geo-shape.asciidoc
+++ b/docs/reference/mapping/types/geo-shape.asciidoc
@@ -502,6 +502,3 @@ synthetic `_source` is in technical preview. Features in technical preview may
 be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
-
-`geo_shape` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration.
diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc
index 8cd30110250bf..cdebe97000d68 100644
--- a/docs/reference/mapping/types/histogram.asciidoc
+++ b/docs/reference/mapping/types/histogram.asciidoc
@@ -79,7 +79,7 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `histogram` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with <<copy-to,`copy_to`>>.
+default configuration.
 
 NOTE: To save space, zero-count buckets are not stored in the histogram doc values.
 As a result, when indexing a histogram field in an index with synthetic source enabled,
diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
index f068916478a78..bafc25a977caa 100644
--- a/docs/reference/mapping/types/ip.asciidoc
+++ b/docs/reference/mapping/types/ip.asciidoc
@@ -161,17 +161,21 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`ip` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `ip` fields and removes duplicates. For example:
+Synthetic source may sort `ip` field values and remove duplicates. For example:
 [source,console,id=synthetic-source-ip-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "ip": { "type": "ip" }
     }
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
index a4be7026dffcd..165d9d7900441 100644
--- a/docs/reference/mapping/types/keyword.asciidoc
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -178,18 +178,22 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`keyword` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-a <<normalizer,`normalizer`>> or <<copy-to,`copy_to`>>.
-
-By default, synthetic source sorts `keyword` fields and removes duplicates.
+Synthetic source may sort `keyword` fields and remove duplicates.
 For example:
 [source,console,id=synthetic-source-keyword-example-default]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "kwd": { "type": "keyword" }
     }
@@ -218,8 +222,16 @@ are preserved. For example:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "kwd": { "type": "keyword", "store": true }
     }
@@ -248,8 +260,16 @@ For example:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "kwd": { "type": "keyword", "ignore_above": 3 }
     }
diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
index d1e1c037e571e..2fba1931a2a29 100644
--- a/docs/reference/mapping/types/numeric.asciidoc
+++ b/docs/reference/mapping/types/numeric.asciidoc
@@ -254,13 +254,21 @@ All numeric fields support <<synthetic-source,synthetic
 together with <<copy-to,`copy_to`>>, or
 with <<doc-values,`doc_values`>> disabled.
 
-Synthetic source always sorts numeric fields. For example:
+Synthetic source may sort numeric field values. For example:
 [source,console,id=synthetic-source-numeric-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "long": { "type": "long" }
     }
@@ -287,8 +295,16 @@ Scaled floats will always apply their scaling factor so:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "f": { "type": "scaled_float", "scaling_factor": 0.01 }
     }
diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc
new file mode 100644
index 0000000000000..f4f1945d21537
--- /dev/null
+++ b/docs/reference/mapping/types/passthrough.asciidoc
@@ -0,0 +1,218 @@
+[[passthrough]]
+=== Pass-through object field type
+++++
+<titleabbrev>Pass-through object</titleabbrev>
+++++
+
+Pass-through objects extend the functionality of <<object, objects>> by allowing to access
+their subfields without including the name of the pass-through object as prefix. For instance:
+
+[source,console]
+--------------------------------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "attributes": {
+        "type": "passthrough", <1>
+        "priority": 10,
+        "properties": {
+          "id": {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}
+
+PUT my-index-000001/_doc/1
+{
+  "attributes" : {  <2>
+    "id": "foo",
+    "zone": 10
+  }
+}
+
+GET my-index-000001/_search
+{
+  "query": {
+    "bool": {
+      "must": [
+        { "match": { "id": "foo" }},  <3>
+        { "match": { "zone": 10 }}
+      ]
+    }
+  }
+}
+
+GET my-index-000001/_search
+{
+  "query": {
+    "bool": {
+      "must": [
+        { "match": { "attributes.id": "foo" }}, <4>
+        { "match": { "attributes.zone": 10 }}
+      ]
+    }
+  }
+}
+
+--------------------------------------------------
+
+<1> An object is defined as pass-through. Its priority (required) is used for conflict resolution.
+<2> Object contents get indexed as usual, including dynamic mappings.
+<3> Sub-fields can be referenced in queries as if they're defined at the root level.
+<4> Sub-fields can also be referenced including the object name as prefix.
+
+[[passthrough-conflicts]]
+==== Conflict resolution
+
+It's possible for conflicting names to arise, for fields that are defined within different scopes:
+
+  a. A pass-through object is defined next to a field that has the same name as one of the pass-through object
+sub-fields, e.g.
++
+[source,console]
+--------------------------------------------------
+PUT my-index-000001/_doc/1
+{
+  "attributes" : {
+    "id": "foo"
+  },
+  "id": "bar"
+}
+--------------------------------------------------
++
+In this case, references to `id` point to the field at the root level, while field `attributes.id`
+can only be accessed using the full path.
+
+  b. Two (or more) pass-through objects are defined within the same object and contain fields with the same name, e.g.
++
+[source,console]
+--------------------------------------------------
+PUT my-index-000002
+{
+  "mappings": {
+    "properties": {
+      "attributes": {
+        "type": "passthrough",
+        "priority": 10,
+        "properties": {
+          "id": {
+            "type": "keyword"
+          }
+        }
+      },
+      "resource.attributes": {
+        "type": "passthrough",
+        "priority": 20,
+        "properties": {
+          "id": {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
++
+In this case, param `priority` is used for conflict resolution, with the higher values taking precedence. In the
+example above, `resource.attributes` has higher priority than `attributes`, so references to `id` point to the field
+within `resource.attributes`. `attributes.id` can still be accessed using its full path.
+
+[[passthrough-dimensions]]
+==== Defining sub-fields as time-series dimensions
+
+It is possible to configure a pass-through field as a container for  <<time-series-dimension,time-series dimensions>>.
+In this case, all sub-fields get annotated with the same parameter under the covers, and they're also
+included in <<dimension-based-routing, routing path>> and <<tsid, tsid>> calculations, thus simplifying
+the <<tsds,TSDS>> setup:
+
+[source,console]
+--------------------------------------------------
+PUT _index_template/my-metrics
+{
+  "index_patterns": ["metrics-mymetrics-*"],
+  "priority": 200,
+  "data_stream": { },
+  "template": {
+    "settings": {
+      "index.mode": "time_series"
+    },
+    "mappings": {
+      "properties": {
+        "attributes": {
+          "type": "passthrough",
+          "priority": 10,
+          "time_series_dimension": true,
+          "properties": {
+            "host.name": {
+              "type": "keyword"
+            }
+          }
+        },
+        "cpu": {
+          "type": "integer",
+          "time_series_metric": "counter"
+        }
+      }
+    }
+  }
+}
+
+POST metrics-mymetrics-test/_doc
+{
+  "@timestamp": "2020-01-01T00:00:00.000Z",
+  "attributes" : {
+    "host.name": "foo",
+    "zone": "bar"
+  },
+  "cpu": 10
+}
+--------------------------------------------------
+// TEST[skip: The @timestamp value won't match an accepted range in the TSDS]
+
+In the example above, `attributes` is defined as a dimension container. Its sub-fields `host.name` (static) and `zone`
+(dynamic) get included in the routing path and tsid, and can be referenced in queries without the `attributes.` prefix.
+
+[[passthrough-flattening]]
+==== Sub-field auto-flattening
+
+Pass-through fields apply <<subobjects-auto-flattening, auto-flattening>> to sub-fields by default, to reduce dynamic
+mapping conflicts. As a consequence, no sub-object definitions are allowed within pass-through fields.
+
+[[passthrough-params]]
+==== Parameters for `passthrough` fields
+
+The following parameters are accepted by `passthrough` fields:
+
+[horizontal]
+
+<<passthrough-conflicts,`priority`>>::
+
+    (Required) used for naming conflict resolution between pass-through fields. The field with the highest value wins.
+    Accepts non-negative integer values.
+
+<<passthrough-dimensions,`time_series_dimension`>>::
+
+    Whether or not to treat sub-fields as <<time-series-dimension,time-series dimensions>>.
+    Accepts `false` (default) or `true`.
+
+<<dynamic,`dynamic`>>::
+
+    Whether or not new `properties` should be added dynamically to an existing object.
+    Accepts `true` (default), `runtime`, `false` and `strict`.
+
+<<enabled,`enabled`>>::
+
+    Whether the JSON value given for the object field should be parsed and indexed (`true`, default)
+    or completely ignored (`false`).
+
+<<properties,`properties`>>::
+
+    The fields within the object, which can be of any <<mapping-types,data type>>, including `object`.
+    New properties may be added to an existing object.
+
+IMPORTANT: If you need to index arrays of objects instead of single objects, read <<nested>> first.
diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc
index 14c5b6098acbe..3b31a1885e5b9 100644
--- a/docs/reference/mapping/types/range.asciidoc
+++ b/docs/reference/mapping/types/range.asciidoc
@@ -247,15 +247,23 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `range` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration. Synthetic `_source` cannot be used with <<doc-values,`doc_values`>> disabled.
+configuration.
 
-Synthetic source always sorts values and removes duplicates for all `range` fields except `ip_range` . Ranges are sorted by their lower bound and then by upper bound. For example:
+Synthetic source may sort `range` field values and remove duplicates for all `range` fields except `ip_range`. Ranges are sorted by their lower bound and then by upper bound. For example:
 [source,console,id=synthetic-source-range-sorting-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "my_range": { "type": "long_range" }
     }
@@ -316,8 +324,16 @@ For example:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "my_range": { "type": "ip_range" }
     }
@@ -352,13 +368,21 @@ Will become:
 // TEST[s/^/{"_source":/ s/\n$/}/]
 
 [[range-synthetic-source-inclusive]]
-Range field vales are always represented as inclusive on both sides with bounds adjusted accordingly. Default values for range bounds are represented as `null`. This is true even if range bound was explicitly provided. For example:
+Range field values are always represented as inclusive on both sides with bounds adjusted accordingly. Default values for range bounds are represented as `null`. This is true even if range bound was explicitly provided. For example:
 [source,console,id=synthetic-source-range-normalization-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "my_range": { "type": "long_range" }
     }
@@ -394,8 +418,16 @@ Default values for range bounds are represented as `null` in synthetic source. T
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "my_range": { "type": "integer_range" }
     }
@@ -429,8 +461,16 @@ Will become:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "my_range": { "type": "date_range" }
     }
diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc
index c0bdc75f13392..3c71389f4cebb 100644
--- a/docs/reference/mapping/types/search-as-you-type.asciidoc
+++ b/docs/reference/mapping/types/search-as-you-type.asciidoc
@@ -266,5 +266,4 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `search_as_you_type` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>.
+default configuration.
diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc
index 07abbff986643..893e2c6cff8ed 100644
--- a/docs/reference/mapping/types/semantic-text.asciidoc
+++ b/docs/reference/mapping/types/semantic-text.asciidoc
@@ -13,25 +13,47 @@ Long passages are <<auto-text-chunking, automatically chunked>> to smaller secti
 The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings.
 You can create the inference endpoint by using the <<put-inference-api>>.
 This field type and the <<query-dsl-semantic-query,`semantic` query>> type make it simpler to perform semantic search on your data.
+If you don't specify an inference endpoint, the <<infer-service-elser,ELSER service>> is used by default.
 
 Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it.
 The {infer} endpoint automatically determines the embedding generation, indexing, and query to use.
 
+If you use the ELSER service, you can set up `semantic_text` with the following API request:
+
 [source,console]
 ------------------------------------------------------------
 PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text"
+      }
+    }
+  }
+}
+------------------------------------------------------------
+
+NOTE: In Serverless, you must create an {infer} endpoint using the <<put-inference-api>> and reference it when setting up `semantic_text` even if you use the ELSER service. 
+
+If you use a service other than ELSER, you must create an {infer} endpoint using the <<put-inference-api>> and reference it when setting up `semantic_text` as the following example demonstrates:
+
+[source,console]
+------------------------------------------------------------
+PUT my-index-000002
 {
   "mappings": {
     "properties": {
       "inference_field": {
         "type": "semantic_text",
-        "inference_id": "my-elser-endpoint"
+        "inference_id": "my-openai-endpoint" <1>
       }
     }
   }
 }
 ------------------------------------------------------------
 // TEST[skip:Requires inference endpoint]
+<1> The `inference_id` of the {infer} endpoint to use to generate embeddings.
 
 
 The recommended way to use semantic_text is by having dedicated {infer} endpoints for ingestion and search.
@@ -40,7 +62,7 @@ After creating dedicated {infer} endpoints for both, you can reference them usin
 
 [source,console]
 ------------------------------------------------------------
-PUT my-index-000002
+PUT my-index-000003
 {
   "mappings": {
     "properties": {
@@ -221,4 +243,5 @@ Notice that both the `semantic_text` field and the source field are updated in t
 `semantic_text` field types have the following limitations:
 
 * `semantic_text` fields are not currently supported as elements of <<nested,nested fields>>.
+* `semantic_text` fields can't currently be set as part of <<dynamic-templates>>.
 * `semantic_text` fields can't be defined as <<multi-fields,multi-fields>> of another field, nor can they contain other fields as multi-fields.
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
index c33af69df5607..b10484fc5ded8 100644
--- a/docs/reference/mapping/types/text.asciidoc
+++ b/docs/reference/mapping/types/text.asciidoc
@@ -134,10 +134,6 @@ The following parameters are accepted by `text` fields:
 
     Whether the field value should be stored and retrievable separately from
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false` (default).
-    This parameter will be automatically set to `true` for TSDB indices
-    (indices that have `index.mode` set to `time_series`)
-    if there is no <<keyword-synthetic-source, `keyword`>>
-    sub-field that supports synthetic `_source`.
 
 <<search-analyzer,`search_analyzer`>>::
 
@@ -177,15 +173,23 @@ a <<keyword-synthetic-source, `keyword`>> sub-field that supports synthetic
 `_source` or if the `text` field sets `store` to `true`. Either way, it may
 not have <<copy-to,`copy_to`>>.
 
-If using a sub-`keyword` field then the values are sorted in the same way as
+If using a sub-`keyword` field, then the values are sorted in the same way as
 a `keyword` field's values are sorted. By default, that means sorted with
 duplicates removed. So:
 [source,console,id=synthetic-source-text-example-default]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": {
         "type": "text",
@@ -233,8 +237,16 @@ are preserved.
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": { "type": "text", "store": true }
     }
diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
index 7d9dffcc82082..2e5bd111122c8 100644
--- a/docs/reference/mapping/types/token-count.asciidoc
+++ b/docs/reference/mapping/types/token-count.asciidoc
@@ -103,5 +103,4 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `token_count` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>.
+default configuration.
diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc
index 8da0fcae80fcd..1d9f927a80ce4 100644
--- a/docs/reference/mapping/types/version.asciidoc
+++ b/docs/reference/mapping/types/version.asciidoc
@@ -63,31 +63,38 @@ The following parameters are accepted by `version` fields:
 [discrete]
 ==== Limitations
 
-This field type isn't optimized for heavy wildcard, regex or fuzzy searches. While those
-type of queries work in this field, you should consider using a regular `keyword` field if
-you strongly rely on these kind of queries.
-
+This field type isn't optimized for heavy wildcard, regex, or fuzzy searches. While those
+types of queries work in this field, you should consider using a regular `keyword` field if
+you strongly rely on these kinds of queries.
 
 [[version-synthetic-source]]
 ==== Synthetic `_source`
 
 IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
-(indices that have `index.mode` set to `time_series`). For other indices
+(indices that have `index.mode` set to `time_series`). For other indices,
 synthetic `_source` is in technical preview. Features in technical preview may
 be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`version` fields support <<synthetic-source,synthetic `_source`>> so long as they don't
-declare <<copy-to,`copy_to`>>.
+`version` fields support <<synthetic-source,synthetic `_source`>> in their
+default configuration..
 
-Synthetic source always sorts `version` fields and removes duplicates. For example:
+Synthetic source may sort `version` field values and remove duplicates. For example:
 [source,console,id=synthetic-source-version-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "versions": { "type": "version" }
     }
diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
index 79fc953051d54..255e34ecd959b 100644
--- a/docs/reference/mapping/types/wildcard.asciidoc
+++ b/docs/reference/mapping/types/wildcard.asciidoc
@@ -133,16 +133,22 @@ The following parameters are accepted by `wildcard` fields:
 
 [[wildcard-synthetic-source]]
 ==== Synthetic `_source`
-`wildcard` fields support <<synthetic-source,synthetic `_source`>> so long as they don't
-declare <<copy-to,`copy_to`>>.
 
-Synthetic source always sorts `wildcard` fields. For example:
+Synthetic source may sort `wildcard` field values. For example:
 [source,console,id=synthetic-source-wildcard-example]
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "card": { "type": "wildcard" }
     }
diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc
index 0690f60495c97..719588cb4b0d0 100644
--- a/docs/reference/migration/index.asciidoc
+++ b/docs/reference/migration/index.asciidoc
@@ -1,5 +1,6 @@
 include::migration_intro.asciidoc[]
 
+* <<migrating-8.17,Migrating to 8.17>>
 * <<migrating-8.16,Migrating to 8.16>>
 * <<migrating-8.15,Migrating to 8.15>>
 * <<migrating-8.14,Migrating to 8.14>>
@@ -18,6 +19,7 @@ include::migration_intro.asciidoc[]
 * <<migrating-8.1,Migrating to 8.1>>
 * <<migrating-8.0,Migrating to 8.0>>
 
+include::migrate_8_17.asciidoc[]
 include::migrate_8_16.asciidoc[]
 include::migrate_8_15.asciidoc[]
 include::migrate_8_14.asciidoc[]
diff --git a/docs/reference/migration/migrate_8_17.asciidoc b/docs/reference/migration/migrate_8_17.asciidoc
new file mode 100644
index 0000000000000..15bc6431c60ba
--- /dev/null
+++ b/docs/reference/migration/migrate_8_17.asciidoc
@@ -0,0 +1,20 @@
+[[migrating-8.17]]
+== Migrating to 8.17
+++++
+<titleabbrev>8.17</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 8.17.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+coming::[8.17.0]
+
+
+[discrete]
+[[breaking-changes-8.17]]
+=== Breaking changes
+
+There are no breaking changes in {es} 8.17.
+
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index ef19fbf4e267d..d01047eac9815 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -3,7 +3,6 @@ Adaptive allocations configuration object.
 If enabled, the number of allocations of the model is set based on the current load the process gets.
 When the load is high, a new model allocation is automatically created (respecting the value of `max_number_of_allocations` if it's set).
 When the load is low, a model allocation is automatically removed (respecting the value of `min_number_of_allocations` if it's set).
-The number of model allocations cannot be scaled down to less than `1` this way.
 If `adaptive_allocations` is enabled, do not set the number of allocations manually.
 end::adaptive-allocation[]
 
diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc
index 8fdc9f2e4f9cb..1e4c5a21d386c 100644
--- a/docs/reference/modules/network.asciidoc
+++ b/docs/reference/modules/network.asciidoc
@@ -153,23 +153,34 @@ The only requirements are that each node must be:
 
 * Accessible at its transport publish address by all other nodes in its
   cluster, and by any remote clusters that will discover it using
-  <<sniff-mode>>.
+  <<sniff-mode,sniff mode>>.
 
 Each node must have its own distinct publish address.
 
 If you specify the transport publish address using a hostname then {es} will
 resolve this hostname to an IP address once during startup, and other nodes
 will use the resulting IP address instead of resolving the name again
-themselves. To avoid confusion, use a hostname which resolves to the node's
-address in all network locations.
+themselves. You must use a hostname such that all of the addresses to which it
+resolves are addresses at which the node is accessible from all other nodes. To
+avoid confusion, it is simplest to use a hostname which resolves to a single
+address.
+
+If you specify the transport publish address using a
+<<network-interface-values,special value>> then {es} will resolve this value to
+a single IP address during startup, and other nodes will use the resulting IP
+address instead of resolving the value again themselves. You must use a value
+such that all of the addresses to which it resolves are addresses at which the
+node is accessible from all other nodes. To avoid confusion, it is simplest to
+use a value which resolves to a single address. It is usually a mistake to use
+`0.0.0.0` as a publish address on hosts with more than one network interface.
 
 ===== Using a single address
 
 The most common configuration is for {es} to bind to a single address at which
-it is accessible to clients and other nodes. In this configuration you should
-just set `network.host` to that address. You should not separately set any bind
-or publish addresses, nor should you separately configure the addresses for the
-HTTP or transport interfaces.
+it is accessible to clients and other nodes. To use this configuration, set
+only `network.host` to the desired address. Do not separately set any bind or
+publish addresses. Do not separately specify the addresses for the HTTP or
+transport interfaces.
 
 ===== Using multiple addresses
 
diff --git a/docs/reference/query-dsl/terms-set-query.asciidoc b/docs/reference/query-dsl/terms-set-query.asciidoc
index 2abfe54d53976..27717af3ac171 100644
--- a/docs/reference/query-dsl/terms-set-query.asciidoc
+++ b/docs/reference/query-dsl/terms-set-query.asciidoc
@@ -159,12 +159,22 @@ GET /job-candidates/_search
 `terms`::
 +
 --
-(Required, array of strings) Array of terms you wish to find in the provided
+(Required, array) Array of terms you wish to find in the provided
 `<field>`. To return a document, a required number of terms must exactly match
 the field values, including whitespace and capitalization.
 
-The required number of matching terms is defined in the
-`minimum_should_match_field` or `minimum_should_match_script` parameter.
+The required number of matching terms is defined in the `minimum_should_match`,
+`minimum_should_match_field` or `minimum_should_match_script` parameters. Exactly
+one of these parameters must be provided.
+--
+
+`minimum_should_match`::
++
+--
+(Optional) Specification for the number of matching terms required to return
+a document.
+
+For valid values, see <<query-dsl-minimum-should-match, `minimum_should_match` parameter>>.
 --
 
 `minimum_should_match_field`::
diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc
index 235a413df686f..5c7bce8c3fcf0 100644
--- a/docs/reference/query-dsl/text-expansion-query.asciidoc
+++ b/docs/reference/query-dsl/text-expansion-query.asciidoc
@@ -7,6 +7,13 @@
 
 deprecated[8.15.0, This query has been replaced by <<query-dsl-sparse-vector-query>>.]
 
+.Deprecation usage note
+****
+You can continue using `rank_features` fields with `text_expansion` queries in the current version.
+However, if you plan to upgrade, we recommend updating mappings to use the `sparse_vector` field type and <<docs-reindex,reindexing your data>>.
+This will allow you to take advantage of the new capabilities and improvements available in newer versions.
+****
+
 The text expansion query uses a {nlp} model to convert the query text into a list of token-weight pairs which are then used in a query against a
 <<sparse-vector,sparse vector>> or <<rank-features,rank features>> field.
 
diff --git a/docs/reference/query-rules/apis/index.asciidoc b/docs/reference/query-rules/apis/index.asciidoc
index 53d5fc3dc4eee..fbeb477acacb5 100644
--- a/docs/reference/query-rules/apis/index.asciidoc
+++ b/docs/reference/query-rules/apis/index.asciidoc
@@ -23,6 +23,7 @@ Use the following APIs to manage query rulesets:
 * <<put-query-rule>>
 * <<get-query-rule>>
 * <<delete-query-rule>>
+* preview:[] <<test-query-ruleset>>
 
 include::put-query-ruleset.asciidoc[]
 include::get-query-ruleset.asciidoc[]
@@ -31,4 +32,5 @@ include::delete-query-ruleset.asciidoc[]
 include::put-query-rule.asciidoc[]
 include::get-query-rule.asciidoc[]
 include::delete-query-rule.asciidoc[]
+include::test-query-ruleset.asciidoc[]
 
diff --git a/docs/reference/query-rules/apis/test-query-ruleset.asciidoc b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc
new file mode 100644
index 0000000000000..4a670645cea6e
--- /dev/null
+++ b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc
@@ -0,0 +1,133 @@
+[role="xpack"]
+[[test-query-ruleset]]
+=== Test query ruleset
+
+++++
+<titleabbrev>Tests query ruleset</titleabbrev>
+++++
+
+Evaluates match criteria against a query ruleset to identify the rules that would match that criteria.
+
+preview::[]
+
+[[test-query-ruleset-request]]
+==== {api-request-title}
+
+`POST _query_rules/<ruleset_id>/_test`
+
+[[test-query-ruleset-prereq]]
+==== {api-prereq-title}
+
+Requires the `manage_search_query_rules` privilege.
+
+[[test-query-ruleset-path-params]]
+==== {api-path-parms-title}
+
+`<ruleset_id>`::
+(Required, string)
+
+[[test-query-rule-request-body]]
+==== {api-request-body-title}
+
+`match_criteria`::
+(Required, object) Defines the match criteria to apply to rules in the given query ruleset.
+Match criteria should match the keys defined in the `criteria.metadata` field of the rule.
+
+[[test-query-ruleset-response-codes]]
+==== {api-response-codes-title}
+
+`400`::
+The `ruleset_id` or `match_criteria` were not provided.
+
+`404` (Missing resources)::
+No query ruleset matching `ruleset_id` could be found.
+
+[[test-query-ruleset-example]]
+==== {api-examples-title}
+
+To test a ruleset, provide the match criteria that you want to test against:
+
+////
+
+[source,console]
+--------------------------------------------------
+PUT _query_rules/my-ruleset
+{
+    "rules": [
+        {
+            "rule_id": "my-rule1",
+            "type": "pinned",
+            "criteria": [
+                {
+                    "type": "contains",
+                    "metadata": "query_string",
+                    "values": [ "pugs", "puggles" ]
+                }
+            ],
+            "actions": {
+                "ids": [
+                    "id1",
+                    "id2"
+                ]
+            }
+        },
+        {
+            "rule_id": "my-rule2",
+            "type": "pinned",
+            "criteria": [
+                {
+                    "type": "fuzzy",
+                    "metadata": "query_string",
+                    "values": [ "rescue dogs" ]
+                }
+            ],
+            "actions": {
+                "docs": [
+                    {
+                        "_index": "index1",
+                        "_id": "id3"
+                    },
+                    {
+                        "_index": "index2",
+                        "_id": "id4"
+                    }
+                ]
+            }
+        }
+    ]
+}
+--------------------------------------------------
+// TESTSETUP
+
+[source,console]
+--------------------------------------------------
+DELETE _query_rules/my-ruleset
+--------------------------------------------------
+// TEARDOWN
+
+////
+
+[source,console]
+----
+POST _query_rules/my-ruleset/_test
+{
+    "match_criteria": {
+        "query_string": "puggles"
+    }
+}
+----
+
+A sample response:
+
+[source,console-result]
+----
+{
+    "total_matched_rules": 1,
+    "matched_rules": [
+        {
+            "ruleset_id": "my-ruleset",
+            "rule_id": "my-rule1"
+        }
+    ]
+}
+----
diff --git a/docs/reference/reference-architectures.asciidoc b/docs/reference/reference-architectures.asciidoc
new file mode 100644
index 0000000000000..4fb25d658391d
--- /dev/null
+++ b/docs/reference/reference-architectures.asciidoc
@@ -0,0 +1,74 @@
+[[reference-architectures]]
+= Reference Architectures
+
+Elasticsearch Reference Architectures serve as essential blueprints for deploying, managing, and optimizing Elasticsearch clusters tailored to different use cases. These architectures provide standardized, proven solutions that help users with best practices for infrastructure setup, data ingestion, indexing, search performance, and high availability. Whether you're handling logs, metrics, or sophisticated search applications, these reference architectures ensure scalability, reliability, and efficient resource utilization. By leveraging these guidelines, organizations can confidently deploy Elasticsearch, achieving optimal performance while minimizing risks and complexities.
+
+TIP: You can host {es} on your own hardware or send your data to {es} on {ecloud} or serverless. 
+
+These reference architectures are recommendations and should be adapted to fit your specific environment and needs. Each solution can vary based on the unique requirements and conditions of your setup.
+
+[discrete]
+[[reference-architectures-time-series-2]]
+=== Architectures
+
+[cols="50, 50"]
+|===
+| *Architecture* | *Use when*
+
+| <<multi-region-two-datacenter-architecture>>
+
+image:reference-architectures/images/multi-region-two-datacenter.png[Image showing a multi-region two datacenter architecture]
+
+a| 
+You want to:
+
+* Monitor the performance and health of their applications in real-time
+* Provide insights and alerts to ensure optimal performance and quick issue resolution.
+
+| <<elastic-cloud-architecture>>
+
+image:reference-architectures/images/elastic-cloud-architecture.png[Image showing a Elastic Cloud Hot-Frozen Architecture]
+
+a| 
+You want to:
+
+* Ipsum lorem 
+* Lorem ipsum
+
+| <<self-managed-single-datacenter>>
+
+image:reference-architectures/images/single-datacenter.png[Image showing a single datacenter architecture]
+
+a| 
+You want to:
+
+* TBD
+* TBD.
+
+| <<three-availability-zones>>
+
+image:reference-architectures/images/three-availability-zone.png[Image showing a three Availability zone architecture]
+
+a| 
+You want to:
+
+* TBD
+* TBD
+|
+|===
+
+[discrete]
+[[reference-architectures-ingest-architectures]]
+=== Ingest Architectures
+
+Additionally, we have architectures specifically tailored to the ingestion portion of your architecture and these can be found at, https://www.elastic.co/guide/en/ingest/current/use-case-arch.html[Ingest Architectures]
+
+include::reference-architectures/components.asciidoc[]
+
+include::reference-architectures/multi-region-two-datacenter-architecture.asciidoc[]
+
+include::reference-architectures/elastic-cloud-architecture.asciidoc[]
+
+include::reference-architectures/self-managed-single-datacenter.asciidoc[]
+
+include::reference-architectures/three-availability-zones.asciidoc[]
diff --git a/docs/reference/reference-architectures/components.asciidoc b/docs/reference/reference-architectures/components.asciidoc
new file mode 100644
index 0000000000000..4950cbe2a04c0
--- /dev/null
+++ b/docs/reference/reference-architectures/components.asciidoc
@@ -0,0 +1,35 @@
+[[reference-architecture-components]]
+== Reference Architecture Components
+
+This page provides an overview of the main node types in Elasticsearch as they relate to the reference architectures. Each node type serves a specific function within the Elasticsearch cluster, contributing to its overall performance and reliability. Understanding these node types is crucial for designing, managing, and optimizing your Elasticsearch deployment.
+
+[discrete]
+[[components-node-type]]
+=== Node types
+
+[cols="1,1,3", options="header"]
+|===
+| Component | Icon | Description
+
+| Master Node
+| image:images/master.png[Image showing a master node]
+| Responsible for cluster-wide settings and state, including index metadata and node information. Ensures cluster health by managing node joining and leaving.
+
+| Data Node
+| image:images/hot.png[Image showing a master node]
+image:images/frozen.png[Image showing a master node]
+| Stores data and performs CRUD, search, and aggregations. High I/O, CPU, and memory requirements.
+
+| Ingest Node
+| ⛓️
+| Preprocesses documents before indexing, applying transformations via ingest pipelines like removing fields, enriching data, etc.
+
+| Coordinating Node
+| 🔄
+| Handles client requests, routes them to appropriate nodes, gathers and reduces results. Any node can act as a coordinating node.
+
+| Machine Learning Node
+| image:images/machine-learning.png[Image showing a master node]
+| Executes machine learning jobs, including anomaly detection, data frame analysis, and inference.
+
+|===
\ No newline at end of file
diff --git a/docs/reference/reference-architectures/elastic-cloud-architecture.asciidoc b/docs/reference/reference-architectures/elastic-cloud-architecture.asciidoc
new file mode 100644
index 0000000000000..79f7ab22e7b0e
--- /dev/null
+++ b/docs/reference/reference-architectures/elastic-cloud-architecture.asciidoc
@@ -0,0 +1,177 @@
+[[elastic-cloud-architecture]]
+== Time Series HA Architecture: Elastic Cloud - Hot-Frozen, Multi-Availability Zone Architecture
+
+The Hot-Frozen Elasticsearch cluster architecture is cost optimized for large time-series datasets while keeping all of the data **fully searchable**. There is no need to "re-hydrate" archived data. In this architecture, the hot tier is primarily used for indexing and immediate searching (1-3 days) with a majority of the search being handled by the frozen tier. Since the data is moved to searchable snapshots in an object store, the cost of keeping all of the data searchable is dramatically reduced.
+
+This architecture is ideal for observability use cases. The architecture includes all the necessary components of the Elastic Stack and is not intended for sizing workloads, but rather as a basis to ensure the architecture you deploy is foundationally ready to handle any desired workload with resiliency. This architecture shows a very high level representation of data flow. For more details on that, see this <LINK>.
+
+[discrete]
+[[cloud-hot-use-case]]
+=== Use Case
+
+This architecture is intended for organizations that need to:
+
+* Monitor the performance and health of their applications in real-time, including the creation and tracking of SLOs (Service Level Objectives).
+* Provide insights and alerts to ensure optimal performance and quick issue resolution for applications.
+* Apply Machine Learning and Artificial Intelligence to assist SREs and Application Teams in dealing with the large amount of data in this type of use case.
+* Generative ai? OTEL collector? Highlight the stuff that sells…
+
+
+[discrete]
+[[cloud-hot-frozen-architecture]]
+=== Architecture
+
+image::images/elastic-cloud-architecture.png["An Elastic Cloud Architecture"]
+
+The diagram illustrates an Elasticsearch cluster deployed in Elastic Cloud across 3 availability zones (AZ). For production we recommend a minimum of 2 availability zones and 3 availability zones for mission critical applications. See https://www.elastic.co/guide/en/cloud/current/ec-planning.html[Plan for Production] for more details. 
+
+TIP: that even if the cluster is deployed across only two AZ, a third master node is still required for quorum voting and will be created automatically in the third AZ.    
+
+The number of data nodes shown for each tier (hot and frozen) is illustrative and would be scaled up depending on ingest volume and retention period (see the example below).   Hot nodes contain both primary and replica shards.  By default, primary and replica shards are always guaranteed to be in different availability zones.   Frozen nodes rely on a large high-speed cache and retrieve data from the Snapshot Store as needed.
+
+Machine learning nodes are optional but highly recommended for large scale time series use cases since the amount of data quickly becomes too difficult to analyze without applying techniques such as machine learning based anomaly detection.
+
+The following section discusses the recommended Elastic Cloud instance types and underlying hardware type for each cloud provider for the hot-frozen deployment illustrated in the diagram above.
+
+[discrete]
+[[recommended-hardware]]
+=== Recommended Hardware Specifications
+Elastic Cloud allows you to deploy clusters in AWS, Azure and Google Cloud.   Available hardware types and configurations vary across all three cloud providers but each provides instance types that meet our recommendations for the node types used in this architecture:
+
+* **Data - Hot:** since the hot tier is responsible for ingest, search and force-merge (when creating the searchable snapshots to roll data over to the frozen tier), cpu-optimized nodes with solid state drives are strongly recommended. Hot nodes should have a disk:memory ratio no higher than 45:1 and the vCPU:RAM ratio should be a minimum of 0.500.
+* **Data - Frozen:** the frozen tier uses a local cache to hold data from the Snapshot Store in the cloud providers' object store.   For the best query performance in the frozen tier, frozen nodes should use solid state drives with a disk:memory ratio of at least 75:1 and a vCPU:RAM ratio of at least 0.133.
+* **Machine Learning:** Storage is not a key factor for ML nodes, however CPU and memory are important considerations. Each of our recommended instance types for machine learning have a vCPU:RAM ratio of at least 0.250.
+* **Master:**  Storage is not a key factor for master nodes, however CPU and memory are important considerations. Each of our recommended instance types for master nodes have a vCPU:RAM ratio of at least 0.500.
+* **Kibana:** Storage is not a key factor for kibana nodes, however CPU and memory are important considerations. Each of our recommended instance types for kibana nodes have a vCPU:RAM ratio of at least 0.500.
+
+The following table shows our specific recommendations for nodes in this architecture.
+
+[cols="10, 30, 30, 30"]
+|===
+| *Type* | *AWS Instance/Type* | *Azure Instance/Type* | *GCP Instance/Type*
+|image:images/hot.png["An Elastic Cloud Architecture"] | aws.es.datahot.c6gd
+c6gd |azure.es.datahot.fsv2
+f32sv2|gcp.es.datahot.n2.68x32x45
+
+N2
+|image:images/frozen.png["An Elastic Cloud Architecture"] 
+| aws.es.datafrozen.i3en
+
+i3en
+ |
+azure.es.datafrozen.edsv4
+
+e8dsv4
+|
+gcp.es.datafrozen.n2.68x10x95
+
+N2
+|image:images/machine-learning.png["An Elastic Cloud Architecture"] 
+| aws.es.ml.m6gd
+
+m6gd
+|
+azure.es.ml.fsv2
+
+f32sv2
+|
+gcp.es.ml.n2.68x32x45
+
+N2
+|image:images/master.png["An Elastic Cloud Architecture"] 
+| aws.es.master.c6gd
+
+c6gd
+|
+azure.es.master.fsv2
+
+f32sv2
+|
+gcp.es.master.n2.68x32x45
+
+N2
+|image:images/kibana.png["An Elastic Cloud Architecture"] 
+| aws.kibana.c6gd
+
+c6gd
+|
+azure.kibana.fsv2
+
+f32sv2
+|
+gcp.kibana.n2.68x32x45
+
+N2|
+|===
+
+For more details on these instance types, see our documentation on Elastic Cloud hardware for https://www.elastic.co/guide/en/cloud/current/ec-default-aws-configurations.html[AWS], https://www.elastic.co/guide/en/cloud/current/ec-default-azure-configurations.html[Azure] and https://www.elastic.co/guide/en/cloud/current/ec-default-gcp-configurations.html[GCP].
+
+[discrete]
+[[cloud-hot-frozen-example-configuration]]
+=== Example configuration
+
+Based on these hardware recommendations, here is a sample configuration for an ingest rate of 1TB/day with an ILM policy of 1 day in the hot tier and 89 days in the frozen tier for a total of 90 days of searchable data.   Note that the differences in the Hot and Frozen node RAM are due to slight differences in the underlying cloud provider instance types.
+
+[discrete]
+[[aws-configuration]]
+==== AWS Configuration
+* Hot tier: 120G RAM (1 60G RAM node x 2 availability zones)
+* Frozen tier: 120G RAM (1 60G RAM node x 2 availability zones)
+* Machine learning: 128G RAM (1 64G node x 2 availability zones)
+* Master nodes: 24G RAM (8G node x 3 availability zones)
+* Kibana: 16G RAM (16G node x 1 availability zone)
+
+[discrete]
+[[azure-configuration]]
+==== Azure Configuration
+* Hot tier: 120G RAM (1 60G RAM node x 2 availability zones)
+* Frozen tier: 120G RAM (1 60G RAM node x 2 availability zones)
+* Machine learning: 128G RAM (1 64G node x 2 availability zones)
+* Master nodes: 24G RAM (8G node x 3 availability zones)
+* Kibana: 16G RAM (16G node x 1 availability zone)
+
+
+[discrete]
+[[gcp-configuration]]
+==== GCP Configuration
+
+* Hot tier: 128G RAM (1 64G RAM node x 2 availability zones)
+* Frozen tier: 128G RAM (1 64G RAM node x 2 availability zones)
+* Machine learning: 128G RAM (1 64G node x 2 availability zones)
+* Master nodes: 24G RAM (8G node x 3 availability zones)
+* Kibana: 16G RAM (16G node x 1 availability zone)
+
+
+[discrete]
+[[cloud-hot-frozen-considerations]]
+=== Important Considerations
+
+The following list are important conderations for this architecture:
+
+* **Time Series Data Updates:**
+** Typically, time series use cases are append only and there is rarely a need to update documents once they have been ingested into Elasticsearch. The frozen tier is read-only so once data rolls over to the frozen tier documents can no longer be updated.  If there is a need to update documents for some part of the data lifecycle, that will require either a larger hot tier or the introduction of a warm tier to cover the time period needed for document updates.
+* **Multi-AZ Frozen Tier:**
+** When using the frozen tier for storing data for regulatory purposes (e.g. one or more years), we typically recommend a single availability zone. However, since this architecture relies on the frozen tier for most of the search capabilities, we recommend at least two availability zones to ensure that there will be data nodes available in the event of an AZ failure.
+* **Shard Management:**
+** The most important foundational step to maintaining performance as you scale is proper shard sizing, location, count, and shard distribution. For a complete understanding of what shards are and how they should be used please review https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html[this documentation page].
+*** *Sizing:* Maintain shard sizes within https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html#shard-size-recommendation[recommended ranges] and aim for an optimal number of shards.
+*** *Distribution:* In a distributed system, any distributed process is only as fast as the slowest node. As a result, it is optimal to maintain indexes with a primary shard count that is a multiple of the node count in a given tier. This creates even distribution of processing and prevents hotspots.
+**** Shard distribution should be enforced using the https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html#avoid-node-hotspots[‘total shards per node’] index level setting 
+
+TIP: For consistent index level settings is it easiest to use index lifecycle management with index templates, please see the section below for more detail.
+
+* **Architecture Variant - adding a Cold Tier**
+** The hot-frozen architecture works well for most time-series use cases.   However, when there is a need for more frequent, low-latency searches, introducing a cold tier may be required.   Some common examples include detection rule lookback for security use cases or complex custom dashboards.  The ILM policy for the example Hot-Frozen architecture above could be modified from 1 day  in hot, 89 in frozen to 1 day in hot, 7 days in cold, and 82 days in frozen.   Cold nodes fully mount a searchable snapshot for primary shards; replica shards are not needed for reliability. In the event of a failure, cold tier nodes can recover data from the underlying snapshot instead.  See https://www.elastic.co/guide/en/elasticsearch/reference/current/data-tiers.html[Data tiers] for more details on Elasticsearch data tiers. Note: our Data tiers docs may be slightly at odds with the concept of hot/frozen or hot/cold/frozen.   Should they be updated?
+* **Limitations of this architecture.**
+** This architecture is a high-availability Elasticsearch architecture.  It is not intended as a Disaster Recovery architecture since it is deployed across Availability Zones in a single cloud region. This architecture can be enhanced for Disaster Recovery by adding a second deployment in another cloud region. Details on Disaster Recovery for Elasticsearch can be found here.
+
+[discrete]
+[[cloud-hot-frozen-resources]]
+=== Resources and references
+
+* <<shard-size-best-practices,Size your shards>>
+* https://www.elastic.co/guide/en/cloud/current/ec-getting-started.html[Elastic Cloud (Elasticsearch Service)]
+* https://www.elastic.co/guide/en/cloud/current/ec-prepare-production.html[Elastic Cloud - Preparing a deployment for production]
+* https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[Elasticsearch Documentation]
+* https://www.elastic.co/guide/en/kibana/current/index.html[Kibana Documentation]
+* https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html[Size your shards]
\ No newline at end of file
diff --git a/docs/reference/reference-architectures/images/elastic-cloud-architecture.png b/docs/reference/reference-architectures/images/elastic-cloud-architecture.png
new file mode 100644
index 0000000000000..d1664a4ac8c83
Binary files /dev/null and b/docs/reference/reference-architectures/images/elastic-cloud-architecture.png differ
diff --git a/docs/reference/reference-architectures/images/frozen.png b/docs/reference/reference-architectures/images/frozen.png
new file mode 100644
index 0000000000000..a965e25368166
Binary files /dev/null and b/docs/reference/reference-architectures/images/frozen.png differ
diff --git a/docs/reference/reference-architectures/images/hot.png b/docs/reference/reference-architectures/images/hot.png
new file mode 100644
index 0000000000000..cd8bad68fdeee
Binary files /dev/null and b/docs/reference/reference-architectures/images/hot.png differ
diff --git a/docs/reference/reference-architectures/images/kibana.png b/docs/reference/reference-architectures/images/kibana.png
new file mode 100644
index 0000000000000..632a118ec9066
Binary files /dev/null and b/docs/reference/reference-architectures/images/kibana.png differ
diff --git a/docs/reference/reference-architectures/images/machine-learning.png b/docs/reference/reference-architectures/images/machine-learning.png
new file mode 100644
index 0000000000000..d59c6422647bc
Binary files /dev/null and b/docs/reference/reference-architectures/images/machine-learning.png differ
diff --git a/docs/reference/reference-architectures/images/master.png b/docs/reference/reference-architectures/images/master.png
new file mode 100644
index 0000000000000..948afe3715a50
Binary files /dev/null and b/docs/reference/reference-architectures/images/master.png differ
diff --git a/docs/reference/reference-architectures/images/multi-region-two-datacenter.png b/docs/reference/reference-architectures/images/multi-region-two-datacenter.png
new file mode 100644
index 0000000000000..caffcad308015
Binary files /dev/null and b/docs/reference/reference-architectures/images/multi-region-two-datacenter.png differ
diff --git a/docs/reference/reference-architectures/images/single-datacenter.png b/docs/reference/reference-architectures/images/single-datacenter.png
new file mode 100644
index 0000000000000..15210d6c43826
Binary files /dev/null and b/docs/reference/reference-architectures/images/single-datacenter.png differ
diff --git a/docs/reference/reference-architectures/images/three-availability-zone.png b/docs/reference/reference-architectures/images/three-availability-zone.png
new file mode 100644
index 0000000000000..f9ea160a94038
Binary files /dev/null and b/docs/reference/reference-architectures/images/three-availability-zone.png differ
diff --git a/docs/reference/reference-architectures/multi-region-two-datacenter-architecture.asciidoc b/docs/reference/reference-architectures/multi-region-two-datacenter-architecture.asciidoc
new file mode 100644
index 0000000000000..0013ee77942ce
--- /dev/null
+++ b/docs/reference/reference-architectures/multi-region-two-datacenter-architecture.asciidoc
@@ -0,0 +1,47 @@
+[[multi-region-two-datacenter-architecture]]
+== Time Series HA Architecture: Multi-Region With Two Datacenters
+
+This article defines a scalable and highly available architecture for Elasticsearch using two datacenters in separate geographical regions. The architecture includes all the necessary components of the Elastic Stack and is not intended for sizing workloads, but rather as a basis to ensure the architecture you deploy is foundationally ready to handle any desired workload with resiliency. This architecture does include very high level representations of data flow, but the implementation of which will be included in subsequent documentation.
+
+[discrete]
+[[multi-region-use-case]]
+=== Use Case
+
+This architecture is intended for organizations that need to: 
+
+* Monitor the performance and health of their applications in real-time
+* Provide insights and alerts to ensure optimal performance and quick issue resolution for applications
+
+[discrete]
+[[multi-region-architecture]]
+=== Architecture
+
+image::images/multi-region-two-datacenter.png["A multi-region time-series architecture across two datacenters"]
+
+[discrete]
+[[multi-region-considerations]]
+=== Important Considerations
+
+The following list are important conderations for this architecture:
+
+* **Shard Management** 
+** The most important foundational step to maintaining performance as you scale is proper shard sizing, location, count, and shard distribution. For a complete understanding of what shards are and how they should be used please review, https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[Shard sizing].
+*** **Sizing:** Maintain shard sizes within recommended ranges and aim for an optimal number of shards.
+*** **Distribution:** In a distributed system, any distributed process is only as fast as the slowest node. As a result, it is optimal to maintain indexes with a primary shard count that is a multiple of the node count in a given tier. This creates even distribution of processing and prevents hotspots.
+**** Shard distribution should be enforced using the https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html#avoid-node-hotspots['total shards per node'] index level setting 
+
+TIP: For consistent index level settings is it easiest to use index lifecycle management with index templates, please see the section below for more detail.
+
+*** **Shard allocation awareness:** To prevent both a primary and a replica from being copied to the same zone, or in this case the same pod, you can use https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#shard-allocation-awareness[shard allocation awareness] and define a simple attribute in the elaticsearch.yaml file on a per-node basis to make Elasticsearch aware of the physical topology and route shards appropriately. In deployment models with multiple availability zones, AZ's would be used in place of pod location.
+
+*** **Disadvantages of this architecture**
+**** No region resilience
+
+[discrete]
+[[multi-region-resources]]
+=== Resources and references
+
+* <<shard-size-best-practices,Size your shards>>
+* https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[Elasticsearch Documentation]
+* https://www.elastic.co/guide/en/kibana/current/index.html[Kibana Documentation]
+
diff --git a/docs/reference/reference-architectures/self-managed-single-datacenter.asciidoc b/docs/reference/reference-architectures/self-managed-single-datacenter.asciidoc
new file mode 100644
index 0000000000000..cdf7f0bb50500
--- /dev/null
+++ b/docs/reference/reference-architectures/self-managed-single-datacenter.asciidoc
@@ -0,0 +1,98 @@
+[[self-managed-single-datacenter]]
+== Elastic Highly Available Architecture: Self Managed -  Single Datacenter
+
+This architecture ensures high availability during normal operations and node maintenance. It includes all necessary Elastic Stack components but is not intended for workload sizing. Use this pattern as a foundation and extend it to meet your specific needs. While it represents data flow for context, implementations may vary. Key design elements include the number and location of master nodes, data nodes, zone awareness, and shard allocation strategy. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/high-availability-cluster-design-large-clusters.html#high-availability-cluster-design-two-zones[Resilience in larger clusters - Two-zone clusters]. This design does not cover cross-region (geographically diverse) disaster recovery.
+
+While this architecture does include a representation of a data flow, this is being provided for contextual understanding and may differ from implementation to implementation.  The critical portion of the design is the number and location of master nodes, the location of data nodes, zone awareness and the shard allocation strategy.  For additional information see this https://www.elastic.co/guide/en/elasticsearch/reference/current/high-availability-cluster-design-large-clusters.html#high-availability-cluster-design-two-zones[reference].
+This design does not address cross region (i.e. geographically diverse) disaster recovery.  
+
+
+[discrete]
+[[single-datacenter-use-case]]
+=== Use Case
+
+This architecture is intended for organizations that need to:
+
+* Store data that is written once and not updated (e.g. logs, metrics or even an accounting ledger where balance updates are done via additional offsetting entries)
+* Be resilient to hardware failures
+* Ensure availability during operational maintenance of any given (zone i.e. POD in the diagram)
+* Maintain a single copy of the data during maintenance
+* Leverage a Frozen Data tier as part of the Information Lifecycle
+* Leverage a Snapshot Repository for additional recovery options 
+
+[discrete]
+[[single-datacenter-architecture]]
+=== Architecture
+
+image::images/single-datacenter.png["A self hosted single datacenter deployment"]
+
+[discrete]
+[[single-datacenter-considerations]]
+=== Important Considerations
+
+The following list are important conderations for this architecture:
+
+* **Operate**
+
+** Maintenance will be done only on one POD at a time.
+
+** A yellow cluster state is acceptable during maintenance.  (This will be due to replica shards being unassigned.)
+
+* **Sample Initial Settings / Configuration:**
+
+** 3 Master Nodes (1 that is voting only) - Note: an odd number of initial master nodes is required.
+
+** 4 Hot Data Nodes; 2 Frozen Nodes
+
+** 1 Primary; 1 Replica
+
+** Machine Learning Nodes - Optional (1 per POD-1, 2)
+
+** Index - total_shards_per_node = 1 (assuming there will be always more nodes than shards needed).  This will prevent hot-spotting.  This should; however,  be relaxed to total_shards_per_node = 2 if the number of nodes and required number of shards are equal or close to equal due to the shard allocation processes being opportunistic. (i.e. if overly aggressive, shards could be placed in a way to create a situation where a shard could not be allocated - and create a yellow cluster state)
+
+** Set up a repository for the frozen tier.
+
+** Set up a  snapshot repository.
+
+* **Shard Management:**
+
+** The most important foundational step to maintaining performance as you scale is proper shard sizing, location, count, and shard distribution. For a complete understanding of what shards are and how they should be used please review https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html[this documentation page].
+
+*** *Sizing:* Maintain shard sizes within https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html#shard-size-recommendation[recommended ranges] and aim for an optimal number of shards.
+
+*** *Distribution:* In a distributed system, any distributed process is only as fast as the slowest node. As a result, it is optimal to maintain indexes with a primary shard count that is a multiple of the node count in a given tier. This creates even distribution of processing and prevents hotspots.
+
+**** Shard distribution should be enforced using the https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html#avoid-node-hotspots[‘total shards per node’] index level setting 
+
+TIP: For consistent index level settings is it easiest to use index lifecycle management with index templates, please see the section below for more detail.
+
+*** *Shard allocation awareness:* To prevent both a primary and a replica from being copied to the same zone, or in this case the same pod, you can use https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#shard-allocation-awareness[shard allocation awareness] and define a simple attribute in the elaticsearch.yaml file on a per-node basis to make Elasticsearch aware of the physical topology and route shards appropriately. In deployment models with multiple availability zones, AZ's would be used in place of pod location.
+
+* **Forced Awareness:**  This should be set in In order to prevent Elastic from trying to create replica shards when a given POD is down for maintenance.
+
+* https://www.elastic.co/guide/en/elasticsearch/reference/8.16/data-tiers.html[ILM (Information Lifecycle Management): Considerations]
+**** Hot:
+***** Use this tier for ingestion. (Note:  we are assuming for this pattern no updates to the data once written).
+***** Use this tier for fastest reads on the most current data.
+**** Warm / Cold - not considered for this pattern.
+**** Frozen:
+***** Data is persisted in a repository; however, it is accessed from the node’s cache.  It may not be as fast as the Hot tier; however, it can still be fast depending on the caching strategy.  
+***** Frozen does not mean slow - it means immutable and saved in durable storage.
+* https://www.elastic.co/guide/en/elasticsearch/reference/8.16/snapshots-take-snapshot.html#automate-snapshots-slm[SLM (Snapshot Lifecycle Management): Considerations]
+* *Limitations of this pattern*
+** No region resilience
+** Only a single copy of (some of … i.e. the most recently written data that is not yet part of a snapshot) data exists during maintenance windows - (Note:  This could be addressed by adding data nodes to POD 3 and setting the sharding strategy to 1 Primary and 2 Replicas)
+** Assumes write once (no updating of documents)
+* **Benefits of this pattern**
+** Reduces cost by leveraging the Frozen tier as soon as that makes sense from an ingest and most frequently read documents perspective
+** Significantly reduces the likelihood of hot-spotting due to the sharding strategy
+** Eliminates network and disk overhead caused by rebalancing attempts that would occur during maintenance due to setting forced awareness.
+
+
+[discrete]
+[[single-datacenter-resources]]
+=== Resources and references
+
+* <<shard-size-best-practices,Size your shards>>
+* https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[Elasticsearch Documentation]
+* https://www.elastic.co/guide/en/kibana/current/index.html[Kibana Documentation]
\ No newline at end of file
diff --git a/docs/reference/reference-architectures/three-availability-zones.asciidoc b/docs/reference/reference-architectures/three-availability-zones.asciidoc
new file mode 100644
index 0000000000000..ef73606a5215a
--- /dev/null
+++ b/docs/reference/reference-architectures/three-availability-zones.asciidoc
@@ -0,0 +1,73 @@
+[[three-availability-zones]]
+== Time Series HA Architecture: Three availability zones
+
+This article outlines a scalable and highly available architecture for Elasticsearch using three availability zones. The architecture encompasses all essential components of the Elastic Stack and serves as a foundational blueprint to ensure your deployment is resilient and ready to handle any desired workload. While this overview includes high-level representations of data flow, detailed implementation will be covered in subsequent documentation.
+
+[discrete]
+[[three-availability-zones-use-case]]
+=== Use Case
+
+This architecture is intended for organizations that need to: 
+
+* Be resilient to hardware failures
+* Ensure availability during operational maintenance of any given (zone i.e. POD in the diagram)
+* Maintain a single copy of the data during maintenance
+* Leverage a Frozen Data tier as part of the Information Lifecycle
+* Leverage a Snapshot Repository for additional recovery options 
+* Be used for data that is written once and not updated (e.g. logs, metrics or even an accounting ledger where balance updates are done via additional offsetting entries)
+
+[discrete]
+[[three-availability-zones-architecture]]
+=== Architecture
+
+image::images/three-availability-zone.png["A three-availability-zones time-series architecture"]
+
+[discrete]
+[[three-availability-zones-considerations]]
+=== Important Considerations
+
+The following list are important conderations for this architecture:
+
+* Maintenance will be done only on one POD at a time.
+* A yellow cluster state is acceptable during maintenance.  (This will be due to replica shards being unassigned.)
+* Sample Initial Settings / Configuration:
+** 3 Master Nodes  6 Hot Data Nodes; 3 Frozen Nodes
+** 1 Primary; 1 Replica
+** Machine Learning Nodes - Optional (1 per POD-1, 2, 3 )
+** Index - total_shards_per_node = 1 (assuming there will be always more nodes than shards needed).  This will prevent hot-spotting.  This should; however,  be relaxed to total_shards_per_node = 2 if the number of nodes and required number of shards are equal or close to equal due to the shard allocation processes being opportunistic. (i.e. if overly aggressive, shards could be placed in a way to create a situation where a shard could not be allocated - and create a yellow cluster state)
+** Set up a repository for the frozen tier.
+* **Shard Management** 
+** The most important foundational step to maintaining performance as you scale is proper shard sizing, location, count, and shard distribution. For a complete understanding of what shards are and how they should be used please review, https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[Shard sizing].
+*** **Sizing:** Maintain shard sizes within recommended ranges and aim for an optimal number of shards.
+*** **Distribution:** In a distributed system, any distributed process is only as fast as the slowest node. As a result, it is optimal to maintain indexes with a primary shard count that is a multiple of the node count in a given tier. This creates even distribution of processing and prevents hotspots.
+**** Shard distribution should be enforced using the https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html#avoid-node-hotspots['total shards per node'] index level setting 
+
+TIP: For consistent index level settings is it easiest to use index lifecycle management with index templates, please see the section below for more detail.
+
+* https://www.elastic.co/guide/en/elasticsearch/reference/8.16/data-tiers.html[ILM (Information Lifecycle Management): Considerations]
+** Hot:
+*** Use this tier for ingestion. (Note:  we are assuming for this pattern no updates to the data once written).
+*** Use this tier for fastest reads on the most current data.
+** Warm / Cold - not considered for this pattern.
+** Frozen:
+*** Data is persisted in a repository; however, it is accessed from the node’s cache.  It may not be as fast as the Hot tier; however, it can still be fast depending on the caching strategy.  
+*** Frozen does not mean slow - it means immutable and saved in durable storage.
+
+* https://www.elastic.co/guide/en/elasticsearch/reference/8.16/snapshots-take-snapshot.html#automate-snapshots-slm[SLM (Snapshot Lifecycle Management): Considerations]
+* *Limitations of this pattern*
+** No region resilience
+** Only a single copy of (some of … i.e. the most recently written data that is not yet part of a snapshot) data exists during maintenance windows - (Note:  This could be addressed by adding data nodes to POD 3 and setting the sharding strategy to 1 Primary and 2 Replicas)
+** Assumes write once (no updating of documents)
+* **Benefits of this pattern**
+** Reduces cost by leveraging the Frozen tier as soon as that makes sense from an ingest and most frequently read documents perspective
+** Significantly reduces the likelihood of hot-spotting due to the sharding strategy
+** Eliminates network and disk overhead caused by rebalancing attempts that would occur during maintenance due to setting forced awareness.
+
+[discrete]
+[[three-availability-zones-resources]]
+=== Resources and references
+
+* <<shard-size-best-practices,Size your shards>>
+* https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[Elasticsearch Documentation]
+* https://www.elastic.co/guide/en/kibana/current/index.html[Kibana Documentation]
+
diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index 6f32b55c49af8..c912b0e62b94d 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -6,6 +6,7 @@
 
 This section summarizes the changes in each release.
 
+* <<release-notes-8.17.0>>
 * <<release-notes-8.16.0>>
 * <<release-notes-8.15.1>>
 * <<release-notes-8.15.0>>
@@ -72,6 +73,7 @@ This section summarizes the changes in each release.
 
 --
 
+include::release-notes/8.17.0.asciidoc[]
 include::release-notes/8.16.0.asciidoc[]
 include::release-notes/8.15.1.asciidoc[]
 include::release-notes/8.15.0.asciidoc[]
diff --git a/docs/reference/release-notes/8.12.0.asciidoc b/docs/reference/release-notes/8.12.0.asciidoc
index bfa99401f41a2..bd0ae032ef0b9 100644
--- a/docs/reference/release-notes/8.12.0.asciidoc
+++ b/docs/reference/release-notes/8.12.0.asciidoc
@@ -11,7 +11,7 @@ Also see <<breaking-changes-8.12,Breaking changes in 8.12>>.
 +
 When using `int8_hnsw` and the default `confidence_interval` (or any `confidence_interval` less than `1.0`) and when
 there are deleted documents in the segments, quantiles may fail to build and prevent merging.
-
++
 This issue is fixed in 8.12.1.
 
 * When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes,
diff --git a/docs/reference/release-notes/8.17.0.asciidoc b/docs/reference/release-notes/8.17.0.asciidoc
new file mode 100644
index 0000000000000..59962fd83e9b7
--- /dev/null
+++ b/docs/reference/release-notes/8.17.0.asciidoc
@@ -0,0 +1,8 @@
+[[release-notes-8.17.0]]
+== {es} version 8.17.0
+
+coming[8.17.0]
+
+Also see <<breaking-changes-8.17,Breaking changes in 8.17>>.
+
+
diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc
index 1e0018f590ac0..c3f6fb43f2ffd 100644
--- a/docs/reference/release-notes/highlights.asciidoc
+++ b/docs/reference/release-notes/highlights.asciidoc
@@ -1,5 +1,6 @@
+[chapter]
 [[release-highlights]]
-== What's new in {minor-version}
+= What's new in {minor-version}
 
 coming::[{minor-version}]
 
@@ -11,7 +12,8 @@ For detailed information about this release, see the <<es-release-notes>> and
 // Add previous release to the list
 Other versions:
 
-{ref-bare}/8.15/release-highlights.html[8.15]
+{ref-bare}/8.16/release-highlights.html[8.16]
+| {ref-bare}/8.15/release-highlights.html[8.15]
 | {ref-bare}/8.14/release-highlights.html[8.14]
 | {ref-bare}/8.13/release-highlights.html[8.13]
 | {ref-bare}/8.12/release-highlights.html[8.12]
@@ -30,11 +32,13 @@ Other versions:
 
 endif::[]
 
+// The notable-highlights tag marks entries that
+// should be featured in the Stack Installation and Upgrade Guide:
 // tag::notable-highlights[]
 
 [discrete]
 [[esql_inlinestats]]
-=== ESQL: INLINESTATS
+== ESQL: INLINESTATS
 This adds the `INLINESTATS` command to ESQL which performs a STATS and
 then enriches the results into the output stream. So, this query:
 
@@ -59,7 +63,7 @@ Produces output like:
 
 [discrete]
 [[always_allow_rebalancing_by_default]]
-=== Always allow rebalancing by default
+== Always allow rebalancing by default
 In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to
 `indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was
 appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has
@@ -71,7 +75,7 @@ version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy al
 
 [discrete]
 [[add_global_retention_in_data_stream_lifecycle]]
-=== Add global retention in data stream lifecycle
+== Add global retention in data stream lifecycle
 Data stream lifecycle now supports configuring retention on a cluster level,
 namely global retention. Global retention \nallows us to configure two different
 retentions:
@@ -85,7 +89,7 @@ data stream lifecycle and it allows any data stream \ndata to be deleted after t
 
 [discrete]
 [[enable_zstandard_compression_for_indices_with_index_codec_set_to_best_compression]]
-=== Enable ZStandard compression for indices with index.codec set to best_compression
+== Enable ZStandard compression for indices with index.codec set to best_compression
 Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to
 best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with
 index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a
@@ -97,29 +101,3 @@ ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput
 // end::notable-highlights[]
 
 
-[discrete]
-[[esql_multi_value_fields_supported_in_geospatial_predicates]]
-=== ESQL: Multi-value fields supported in Geospatial predicates
-Supporting multi-value fields in `WHERE` predicates is a challenge due to not knowing whether `ALL` or `ANY`
-of the values in the field should pass the predicate.
-For example, should the field `age:[10,30]` pass the predicate `WHERE age>20` or not?
-This ambiguity does not exist with the spatial predicates
-`ST_INTERSECTS` and `ST_DISJOINT`, because the choice between `ANY` or `ALL`
-is implied by the predicate itself.
-Consider a predicate checking a field named `location` against a test geometry named `shape`:
-
-* `ST_INTERSECTS(field, shape)` - true if `ANY` value can intersect the shape
-* `ST_DISJOINT(field, shape)` - true only if `ALL` values are disjoint from the shape
-
-This works even if the shape argument is itself a complex or compound geometry.
-
-Similar logic exists for `ST_CONTAINS` and `ST_WITHIN` predicates, but these are not as easily solved
-with `ANY` or `ALL`, because a collection of geometries contains another collection if each of the contained
-geometries is within at least one of the containing geometries. Evaluating this requires that the multi-value
-field is first combined into a single geometry before performing the predicate check.
-
-* `ST_CONTAINS(field, shape)` - true if the combined geometry contains the shape
-* `ST_WITHIN(field, shape)` - true if the combined geometry is within the shape
-
-{es-pull}112063[#112063]
-
diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc
index 957f57ffc9105..27cc1723265c9 100644
--- a/docs/reference/rest-api/usage.asciidoc
+++ b/docs/reference/rest-api/usage.asciidoc
@@ -38,9 +38,10 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 ------------------------------------------------------------
 GET /_xpack/usage
 ------------------------------------------------------------
-// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*/]
+// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*,-esql.functions*/]
 // This response filter removes watcher logging results if they are included
 // to avoid errors in the CI builds.
+// Same for ES|QL functions, that is a long list and quickly evolving.
 
 [source,console-result]
 ------------------------------------------------------------
@@ -210,7 +211,12 @@ GET /_xpack/usage
         "service": "elasticsearch",
         "task_type": "SPARSE_EMBEDDING",
         "count": 1
-      }
+      },
+      {
+        "service": "elasticsearch",
+        "task_type": "TEXT_EMBEDDING",
+        "count": 1
+      },
     ]
   },
   "logstash" : {
diff --git a/docs/reference/run-elasticsearch-locally.asciidoc b/docs/reference/run-elasticsearch-locally.asciidoc
index 1a115ae926ea2..03885132e4050 100644
--- a/docs/reference/run-elasticsearch-locally.asciidoc
+++ b/docs/reference/run-elasticsearch-locally.asciidoc
@@ -20,7 +20,7 @@ Refer to <<elasticsearch-intro-deploy, deployment options>> for a list of produc
 
 Quickly set up {es} and {kib} in Docker for local development or testing, using the https://github.com/elastic/start-local?tab=readme-ov-file#-try-elasticsearch-and-kibana-locally[`start-local` script].
 
-This setup comes with a one-month trial of the Elastic *Platinum* license.
+This setup comes with a one-month trial license that includes all Elastic features.
 After the trial period, the license reverts to *Free and open - Basic*.
 Refer to https://www.elastic.co/subscriptions[Elastic subscriptions] for more information.
 
@@ -84,4 +84,4 @@ Learn about customizing the setup, logging, and more.
 [[local-dev-next-steps]]
 === Next steps
 
-Use our <<quickstart,quick start guides>> to learn the basics of {es}.
\ No newline at end of file
+Use our <<quickstart,quick start guides>> to learn the basics of {es}.
diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 3fed14231808c..5f1a0ccfdd6b4 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -1298,7 +1298,7 @@ One of the `dfs.knn` sections for a shard looks like the following:
         "query" : [
             {
                 "type" : "DocAndScoreQuery",
-                "description" : "DocAndScore[100]",
+                "description" : "DocAndScoreQuery[0,...][0.008961825,...],0.008961825",
                 "time_in_nanos" : 444414,
                 "breakdown" : {
                   "set_min_competitive_score_count" : 0,
diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc
index 54836ac33762d..9306d83c79136 100644
--- a/docs/reference/search/retriever.asciidoc
+++ b/docs/reference/search/retriever.asciidoc
@@ -1,8 +1,6 @@
 [[retriever]]
 === Retriever
 
-preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
-
 A retriever is a specification to describe top documents returned from a
 search. A retriever replaces other elements of the <<search-search, search API>>
 that also return top documents such as <<query-dsl, `query`>> and
@@ -75,7 +73,7 @@ Collapses the top documents by a specified key into a single top document per ke
 ===== Restrictions
 
 When a retriever tree contains a compound retriever (a retriever with two or more child
-retrievers) *only* the query element is allowed.
+retrievers) the <<search-after, search after>> parameter is not supported.
 
 [discrete]
 [[standard-retriever-example]]
@@ -245,12 +243,6 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-window-size]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-filter]
 
-===== Restrictions
-
-An RRF retriever is a compound retriever. Child retrievers may not use
-elements that are restricted by having a compound retriever as part of
-the retriever tree.
-
 [discrete]
 [[rrf-retriever-example-hybrid]]
 ==== Example: Hybrid search
@@ -413,10 +405,6 @@ Applies the specified <<query-dsl-bool-query, boolean query filter>> to the chil
 If the child retriever already specifies any filters, then this top-level filter is applied in conjuction
 with the filter defined in the child retriever.
 
-===== Restrictions
-
-A text similarity re-ranker retriever is a compound retriever. Child retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree.
-
 [discrete]
 [[text-similarity-reranker-retriever-example-cohere]]
 ==== Example: Cohere Rerank
@@ -555,4 +543,3 @@ at the top-level and instead are only allowed as elements of specific retrievers
 * <<request-body-search-terminate-after, `terminate_after`>>
 * <<search-sort-param, `sort`>>
 * <<rescore, `rescore`>>
-* <<search-api-min-score, `min_score`>>
diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc
index 2a676e5fba336..edd3b67e3de04 100644
--- a/docs/reference/search/rrf.asciidoc
+++ b/docs/reference/search/rrf.asciidoc
@@ -1,8 +1,6 @@
 [[rrf]]
 === Reciprocal rank fusion
 
-preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
-
 https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf[Reciprocal rank fusion (RRF)]
 is a method for combining multiple result sets with different relevance indicators into a single result set.
 RRF requires no tuning, and the different relevance indicators do not have to be related to each other to achieve high-quality results.
@@ -95,19 +93,21 @@ The `rrf` retriever supports:
 
 * <<search-aggregations, aggregations>>
 * <<search-from-param, from>>
+* <<search-suggesters, suggesters>>
+* <<highlighting, highlighting>>
+* <<collapse-search-results, collapse>>
+* <<profiling-queries, profiling>>
 
 The `rrf` retriever does not currently support:
 
 * <<search-api-scroll-query-param, scroll>>
-* <<search-api-pit, point in time>>
 * <<search-sort-param, sort>>
 * <<rescore, rescore>>
-* <<search-suggesters, suggesters>>
-* <<highlighting, highlighting>>
-* <<collapse-search-results, collapse>>
-* <<profiling-queries, profiling>>
 
 Using unsupported features as part of a search with an `rrf` retriever results in an exception.
++
+IMPORTANT: It is best to avoid providing a <<search-api-pit, point in time>> as part of the request, as
+RRF creates one internally that is shared by all sub-retrievers to ensure consistent results.
 
 [[rrf-using-multiple-standard-retrievers]]
 ==== Reciprocal rank fusion using multiple standard retrievers
diff --git a/docs/reference/search/search-your-data/ingest-vectors.asciidoc b/docs/reference/search/search-your-data/ingest-vectors.asciidoc
new file mode 100644
index 0000000000000..f288293d2b03a
--- /dev/null
+++ b/docs/reference/search/search-your-data/ingest-vectors.asciidoc
@@ -0,0 +1,141 @@
+[[bring-your-own-vectors]]
+=== Bring your own dense vector embeddings to {es}
+++++
+<titleabbrev>Bring your own dense vectors</titleabbrev>
+++++
+
+This tutorial demonstrates how to index documents that already have dense vector embeddings into {es}.
+You'll also learn the syntax for searching these documents using a `knn` query.
+
+You'll find links at the end of this tutorial for more information about deploying a text embedding model in {es}, so you can generate embeddings for queries on the fly.
+
+[TIP]
+====
+This is an advanced use case.
+Refer to <<semantic-search,Semantic search>> for an overview of your options for semantic search with {es}.
+====
+
+[discrete]
+[[bring-your-own-vectors-create-index]]
+=== Step 1: Create an index with `dense_vector` mapping
+
+Each document in our simple dataset will have:
+
+* A review: stored in a `review_text` field
+* An embedding of that review: stored in a `review_vector` field
+** The `review_vector` field is defined as a <<dense-vector,`dense_vector`>> data type.
+
+[TIP]
+====
+The `dense_vector` type automatically uses `int8_hnsw` quantization by default to reduce the memory footprint required when searching float vectors.
+Learn more about balancing performance and accuracy in <<dense-vector-quantization,Dense vector quantization>>.
+====
+
+[source,console]
+----
+PUT /amazon-reviews
+{
+  "mappings": {
+    "properties": {
+      "review_vector": {
+        "type": "dense_vector",
+        "dims": 8, <1>
+        "index": true, <2>
+        "similarity": "cosine" <3>
+      },
+      "review_text": {
+        "type": "text"
+      }
+    }
+  }
+}
+----
+// TEST SETUP
+<1> The `dims` parameter must match the length of the embedding vector. Here we're using a simple 8-dimensional embedding for readability. If not specified, `dims` will be dynamically calculated based on the first indexed document.
+<2> The `index` parameter is set to `true` to enable the use of the `knn` query.
+<3> The `similarity` parameter defines the similarity function used to compare the query vector to the document vectors. `cosine` is the default similarity function for `dense_vector` fields in {es}.
+
+[discrete]
+[[bring-your-own-vectors-index-documents]]
+=== Step 2: Index documents with embeddings
+
+[discrete]
+==== Index a single document
+
+First, index a single document to understand the document structure.
+
+[source,console]
+----
+PUT /amazon-reviews/_doc/1
+{
+  "review_text": "This product is lifechanging! I'm telling all my friends about it.",
+  "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] <1>
+}
+----
+// TEST
+<1> The size of the `review_vector` array is 8, matching the `dims` count specified in the mapping.
+
+[discrete]
+==== Bulk index multiple documents
+
+In a production scenario, you'll want to index many documents at once using the <<docs-bulk,`_bulk` endpoint>>.
+
+Here's an example of indexing multiple documents in a single `_bulk` request.
+
+[source,console]
+----
+POST /_bulk
+{ "index": { "_index": "amazon-reviews", "_id": "2" } }
+{ "review_text": "This product is amazing! I love it.", "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] }
+{ "index": { "_index": "amazon-reviews", "_id": "3" } }
+{ "review_text": "This product is terrible. I hate it.", "review_vector": [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] }
+{ "index": { "_index": "amazon-reviews", "_id": "4" } }
+{ "review_text": "This product is great. I can do anything with it.", "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] }
+{ "index": { "_index": "amazon-reviews", "_id": "5" } }
+{ "review_text": "This product has ruined my life and the lives of my family and friends.", "review_vector": [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] }
+----
+// TEST[continued]
+
+[discrete]
+[[bring-your-own-vectors-search-documents]]
+=== Step 3: Search documents with embeddings
+
+Now you can query these document vectors using a <<knn-retriever,`knn` retriever>>.
+`knn` is a type of vector search, which finds the `k` most similar documents to a query vector.
+Here we're simply using a raw vector for the query text, for demonstration purposes.
+
+[source,console]
+----
+POST /amazon-reviews/_search
+{
+  "retriever": {
+    "knn": { 
+      "field": "review_vector",
+      "query_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], <1>
+      "k": 2, <2>
+      "num_candidates": 5 <3>
+    }
+  }
+}
+----
+// TEST[skip:flakeyknnerror]
+<1> In this simple example, we're sending a raw vector as the query text. In a real-world scenario, you'll need to generate vectors for queries using an embedding model.
+<2> The `k` parameter specifies the number of results to return.
+<3> The `num_candidates` parameter is optional. It limits the number of candidates returned by the search node. This can improve performance and reduce costs.
+
+[discrete]
+[[bring-your-own-vectors-learn-more]]
+=== Learn more
+
+In this simple example, we're sending a raw vector for the query text.
+In a real-world scenario you won't know the query text ahead of time.
+You'll need to generate query vectors, on the fly, using the same embedding model that generated the document vectors.
+
+For this you'll need to deploy a text embedding model in {es} and use the <<knn-query-top-level-parameters,`query_vector_builder` parameter>>. Alternatively, you can generate vectors client-side and send them directly with the search request.
+
+Learn how to <<semantic-search-deployed-nlp-model,use a deployed text embedding model>> for semantic search.
+
+[TIP]
+====
+If you're just getting started with vector search in {es}, refer to <<semantic-search,Semantic search>>.
+====
diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc
index 70cf9eec121d7..6fb7f1747051f 100644
--- a/docs/reference/search/search-your-data/knn-search.asciidoc
+++ b/docs/reference/search/search-your-data/knn-search.asciidoc
@@ -1149,3 +1149,95 @@ POST product-index/_search
 ----
 //TEST[continued]
 
+[discrete]
+[[dense-vector-knn-search-reranking]]
+==== Oversampling and rescoring for quantized vectors
+
+All forms of quantization will result in some accuracy loss and as the quantization level increases the accuracy loss will also increase.
+Generally, we have found that:
+- `int8` requires minimal if any rescoring
+- `int4` requires some rescoring for higher accuracy and larger recall scenarios. Generally, oversampling by 1.5x-2x recovers most of the accuracy loss.
+- `bbq` requires rescoring except on exceptionally large indices or models specifically designed for quantization. We have found that between 3x-5x oversampling is generally sufficient. But for fewer dimensions or vectors that do not quantize well, higher oversampling may be required.
+
+There are two main ways to oversample and rescore. The first is to utilize the <<rescore, rescore section>> in the `_search` request.
+
+Here is an example using the top level `knn` search with oversampling and using `rescore` to rerank the results:
+
+[source,console]
+--------------------------------------------------
+POST /my-index/_search
+{
+  "size": 10, <1>
+  "knn": {
+    "query_vector": [0.04283529, 0.85670587, -0.51402352, 0],
+    "field": "my_int4_vector",
+    "k": 20, <2>
+    "num_candidates": 50
+  },
+  "rescore": {
+    "window_size": 20, <3>
+    "query": {
+      "rescore_query": {
+        "script_score": {
+          "query": {
+            "match_all": {}
+          },
+          "script": {
+            "source": "(dotProduct(params.queryVector, 'my_int4_vector') + 1.0)", <4>
+            "params": {
+              "queryVector": [0.04283529, 0.85670587, -0.51402352, 0]
+            }
+          }
+        }
+      },
+      "query_weight": 0, <5>
+      "rescore_query_weight": 1 <6>
+    }
+  }
+}
+--------------------------------------------------
+// TEST[skip: setup not provided]
+<1> The number of results to return, note its only 10 and we will oversample by 2x, gathering 20 nearest neighbors.
+<2> The number of results to return from the KNN search. This will do an approximate KNN search with 50 candidates
+per HNSW graph and use the quantized vectors, returning the 20 most similar vectors
+according to the quantized score. Additionally, since this is the top-level `knn` object, the global top 20 results
+will from all shards will be gathered before rescoring. Combining with `rescore`, this is oversampling by `2x`, meaning
+gathering 20 nearest neighbors according to quantized scoring and rescoring with higher fidelity float vectors.
+<3> The number of results to rescore, if you want to rescore all results, set this to the same value as `k`
+<4> The script to rescore the results. Script score will interact directly with the originally provided float32 vector.
+<5> The weight of the original query, here we simply throw away the original score
+<6> The weight of the rescore query, here we only use the rescore query
+
+The second way is to score per shard with the <<query-dsl-knn-query, knn query>> and <<query-dsl-script-score-query, script_score query >>. Generally, this means that there will be more rescoring per shard, but this
+can increase overall recall at the cost of compute.
+
+[source,console]
+--------------------------------------------------
+POST /my-index/_search
+{
+  "size": 10, <1>
+  "query": {
+    "script_score": {
+      "query": {
+        "knn": { <2>
+          "query_vector": [0.04283529, 0.85670587, -0.51402352, 0],
+          "field": "my_int4_vector",
+          "num_candidates": 20 <3>
+        }
+      },
+      "script": {
+        "source": "(dotProduct(params.queryVector, 'my_int4_vector') + 1.0)", <4>
+        "params": {
+          "queryVector": [0.04283529, 0.85670587, -0.51402352, 0]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// TEST[skip: setup not provided]
+<1> The number of results to return
+<2> The `knn` query to perform the initial search, this is executed per-shard
+<3> The number of candidates to use for the initial approximate `knn` search. This will search using the quantized vectors
+and return the top 20 candidates per shard to then be scored
+<4> The script to score the results. Script score will interact directly with the originally provided float32 vector.
diff --git a/docs/reference/search/search-your-data/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-overview.asciidoc
index c0fe7471946f3..9df4026fc6445 100644
--- a/docs/reference/search/search-your-data/retrievers-overview.asciidoc
+++ b/docs/reference/search/search-your-data/retrievers-overview.asciidoc
@@ -1,9 +1,7 @@
 [[retrievers-overview]]
 === Retrievers
 
-preview::[]
-
-A retriever is an abstraction that was added to the Search API in *8.14.0*.
+A retriever is an abstraction that was added to the Search API in *8.14.0* and was made generally available in *8.16.0*.
 This abstraction enables the configuration of multi-stage retrieval pipelines within a single `_search` call.
 This simplifies your search application logic, because you no longer need to configure complex searches via multiple {es} calls or implement additional client-side logic to combine results from different queries.
 
@@ -32,8 +30,7 @@ with different relevance indicators into a single result set.
 An RRF retriever is a *compound retriever*, where its `filter` element is
 propagated to its sub retrievers.
 +
-Sub retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree.
-See the <<rrf-using-multiple-standard-retrievers,RRF documentation>> for detailed examples and information on how to use the RRF retriever.
+
 * <<text-similarity-reranker-retriever,*Text Similarity Re-ranker Retriever*>>. Used for <<semantic-reranking,semantic reranking>>.
 Requires first creating a `rerank` task using the <<put-inference-api,{es} Inference API>>.
 
@@ -72,82 +69,56 @@ When using compound retrievers, only the query element is allowed, which enforce
 [[retrievers-overview-example]]
 ==== Example
 
-The following example demonstrates how using retrievers simplify the composability of queries for RRF ranking.
+The following example demonstrates the powerful queries that we can now compose, and how retrievers simplify this process. We can use any combination of retrievers we want, propagating the
+results of a nested retriever to its parent. In this scenario, we'll make use of all 4 (currently) available retrievers, i.e. `standard`, `knn`, `text_similarity_reranker` and `rrf`.
+We'll first combine the results of a `semantic` query using the `standard` retriever, and that of a `knn` search on a dense vector field, using `rrf` to get the top 100 results.
+Finally, we'll then rerank the top-50 results of `rrf` using the `text_similarity_reranker`
 
 [source,js]
 ----
 GET example-index/_search
 {
   "retriever": {
-    "rrf": {
-      "retrievers": [
-        {
-          "standard": {
-            "query": {
-              "sparse_vector": {
-                "field": "vector.tokens",
-                "inference_id": "my-elser-endpoint",
-                "query": "What blue shoes are on sale?"
+    "text_similarity_reranker": {
+      "retriever": {
+        "rrf": {
+          "retrievers": [
+            {
+                "standard": {
+                  "query": {
+                    "semantic": {
+                      "field": "inference_field",
+                      "query": "state of the art vector database"
+                    }
+                  }
+                }
+            },
+            {
+              "knn": {
+                "query_vector": [
+                  0.54,
+                  ...,
+                  0.245
+                ],
+                "field": "embedding",
+                "k": 10,
+                "num_candidates": 15
               }
             }
-          }
-        },
-        {
-          "standard": {
-            "query": {
-              "match": {
-                "text": "blue shoes sale"
-              }
-            }
-          }
-        }
-      ]
-    }
-  }
-}
-----
-//NOTCONSOLE
-
-This example demonstrates how you can combine different retrieval strategies into a single `retriever` pipeline.
-
-Compare to `RRF` with `sub_searches` approach:
-
-.*Expand* for example
-[%collapsible]
-==============
-
-[source,js]
-----
-GET example-index/_search
-{
-  "sub_searches":[
-    {
-      "query":{
-        "match":{
-          "text":"blue shoes sale"
-        }
-      }
-    },
-    {
-      "query":{
-        "sparse_vector": {
-            "field": "vector.tokens",
-            "inference_id": "my-elser-endoint",
-            "query": "What blue shoes are on sale?"
-          }
+          ],
+          "rank_window_size": 100,
+          "rank_constant": 10
         }
-      }
-  ],
-  "rank":{
-    "rrf":{
-      "rank_window_size":50,
-      "rank_constant":20
+      },
+      "rank_window_size": 50,
+      "field": "description",
+      "inference_text": "what's the best way to create complex pipelines and retrieve documents?",
+      "inference_id": "my-awesome-rerank-model"
     }
   }
 }
 ----
 //NOTCONSOLE
-==============
 
 [discrete]
 [[retrievers-overview-glossary]]
diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
index dbcfbb1b615f9..f881ca87a92e6 100644
--- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
@@ -21,45 +21,11 @@ This tutorial uses the <<inference-example-elser,`elser` service>> for demonstra
 [[semantic-text-requirements]]
 ==== Requirements
 
-To use the `semantic_text` field type, you must have an {infer} endpoint deployed in
-your cluster using the <<put-inference-api>>.
+This tutorial uses the <<infer-service-elser,ELSER service>> for demonstration, which is created automatically as needed. 
+To use the `semantic_text` field type with an {infer} service other than ELSER, you must create an inference endpoint using the <<put-inference-api>>.
 
-[discrete]
-[[semantic-text-infer-endpoint]]
-==== Create the {infer} endpoint
-
-Create an inference endpoint by using the <<put-inference-api>>:
+NOTE: In Serverless, you must create an {infer} endpoint using the <<put-inference-api>> and reference it when setting up `semantic_text` even if you use the ELSER service. 
 
-[source,console]
-------------------------------------------------------------
-PUT _inference/sparse_embedding/my-elser-endpoint <1>
-{
-  "service": "elser", <2>
-  "service_settings": {
-    "adaptive_allocations": { <3>
-      "enabled": true,
-      "min_number_of_allocations": 3,
-      "max_number_of_allocations": 10
-    },
-    "num_threads": 1
-  }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
-<1> The task type is `sparse_embedding` in the path as the `elser` service will
-be used and ELSER creates sparse vectors. The `inference_id` is
-`my-elser-endpoint`.
-<2> The `elser` service is used in this example.
-<3> This setting enables and configures adaptive allocations.
-Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process.
-
-[NOTE]
-====
-You might see a 502 bad gateway error in the response when using the {kib} Console.
-This error usually just reflects a timeout, while the model downloads in the background.
-You can check the download progress in the {ml-app} UI.
-If using the Python client, you can set the `timeout` parameter to a higher value.
-====
 
 [discrete]
 [[semantic-text-index-mapping]]
@@ -75,8 +41,7 @@ PUT semantic-embeddings
   "mappings": {
     "properties": {
       "content": { <1>
-        "type": "semantic_text", <2>
-        "inference_id": "my-elser-endpoint" <3>
+        "type": "semantic_text" <2>
       }
     }
   }
@@ -85,18 +50,14 @@ PUT semantic-embeddings
 // TEST[skip:TBD]
 <1> The name of the field to contain the generated embeddings.
 <2> The field to contain the embeddings is a `semantic_text` field.
-<3> The `inference_id` is the inference endpoint you created in the previous step.
-It will be used to generate the embeddings based on the input text.
-Every time you ingest data into the related `semantic_text` field, this endpoint will be used for creating the vector representation of the text.
+Since no `inference_id` is provided, the <<infer-service-elser,ELSER service>> is used by default.
+To use a different {infer} service, you must create an {infer} endpoint first using the <<put-inference-api>> and then specify it in the `semantic_text` field mapping using the `inference_id` parameter.
 
 [NOTE]
 ====
-If you're using web crawlers or connectors to generate indices, you have to
-<<indices-put-mapping,update the index mappings>> for these indices to
-include the `semantic_text` field. Once the mapping is updated, you'll need to run
-a full web crawl or a full connector sync. This ensures that all existing
-documents are reprocessed and updated with the new semantic embeddings,
-enabling semantic search on the updated data.
+If you're using web crawlers or connectors to generate indices, you have to <<indices-put-mapping,update the index mappings>> for these indices to include the `semantic_text` field.
+Once the mapping is updated, you'll need to run a full web crawl or a full connector sync.
+This ensures that all existing documents are reprocessed and updated with the new semantic embeddings, enabling semantic search on the updated data.
 ====
 
 
@@ -284,6 +245,8 @@ query from the `semantic-embedding` index:
 
 [discrete]
 [[semantic-text-further-examples]]
-==== Further examples
+==== Further examples and reading
 
-If you want to use `semantic_text` in hybrid search, refer to https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[this notebook] for a step-by-step guide.
\ No newline at end of file
+* If you want to use `semantic_text` in hybrid search, refer to https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[this notebook] for a step-by-step guide.
+* For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
+* To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
\ No newline at end of file
diff --git a/docs/reference/search/search-your-data/semantic-search.asciidoc b/docs/reference/search/search-your-data/semantic-search.asciidoc
index 62e41b3eef3de..e0fb8415fee18 100644
--- a/docs/reference/search/search-your-data/semantic-search.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search.asciidoc
@@ -8,6 +8,8 @@ Using an NLP model enables you to extract text embeddings out of text.
 Embeddings are vectors that provide a numeric representation of a text.
 Pieces of content with similar meaning have similar representations.
 
+image::images/semantic-options.svg[Overview of semantic search workflows in {es}]
+
 You have several options for using NLP models in the {stack}:
 
 * use the `semantic_text` workflow (recommended)
@@ -104,7 +106,9 @@ IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer
 
 
 include::semantic-search-semantic-text.asciidoc[]
+include::semantic-text-hybrid-search[]
 include::semantic-search-inference.asciidoc[]
 include::semantic-search-elser.asciidoc[]
 include::cohere-es.asciidoc[]
 include::semantic-search-deploy-model.asciidoc[]
+include::ingest-vectors.asciidoc[]
diff --git a/docs/reference/search/search-your-data/semantic-text-hybrid-search b/docs/reference/search/search-your-data/semantic-text-hybrid-search
new file mode 100644
index 0000000000000..c56b283434df5
--- /dev/null
+++ b/docs/reference/search/search-your-data/semantic-text-hybrid-search
@@ -0,0 +1,254 @@
+[[semantic-text-hybrid-search]]
+=== Tutorial: hybrid search with `semantic_text`
+++++
+<titleabbrev>Hybrid search with `semantic_text`</titleabbrev>
+++++
+
+This tutorial demonstrates how to perform hybrid search, combining semantic search with traditional full-text search.
+
+In hybrid search, semantic search retrieves results based on the meaning of the text, while full-text search focuses on exact word matches. By combining both methods, hybrid search delivers more relevant results, particularly in cases where relying on a single approach may not be sufficient.
+
+The recommended way to use hybrid search in the {stack} is following the `semantic_text` workflow. This tutorial uses the <<inference-example-elser,`elser` service>> for demonstration, but you can use any service and its supported models offered by the {infer-cap} API.
+
+[discrete]
+[[semantic-text-hybrid-infer-endpoint]]
+==== Create the {infer} endpoint
+
+Create an inference endpoint by using the <<put-inference-api>>:
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/my-elser-endpoint <1>
+{
+  "service": "elser", <2>
+  "service_settings": {
+    "adaptive_allocations": { <3>
+      "enabled": true,
+      "min_number_of_allocations": 3,
+      "max_number_of_allocations": 10
+    },
+    "num_threads": 1
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The task type is `sparse_embedding` in the path as the `elser` service will
+be used and ELSER creates sparse vectors. The `inference_id` is
+`my-elser-endpoint`.
+<2> The `elser` service is used in this example.
+<3> This setting enables and configures adaptive allocations.
+Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process.
+
+[NOTE]
+====
+You might see a 502 bad gateway error in the response when using the {kib} Console.
+This error usually just reflects a timeout, while the model downloads in the background.
+You can check the download progress in the {ml-app} UI.
+====
+
+[discrete]
+[[hybrid-search-create-index-mapping]]
+==== Create an index mapping for hybrid search
+
+The destination index will contain both the embeddings for semantic search and the original text field for full-text search. This structure enables the combination of semantic search and full-text search.
+
+[source,console]
+------------------------------------------------------------
+PUT semantic-embeddings
+{
+  "mappings": {
+    "properties": {
+      "semantic_text": { <1>
+        "type": "semantic_text", 
+        "inference_id": "my-elser-endpoint" <2>
+      },
+      "content": { <3>
+        "type": "text",
+        "copy_to": "semantic_text" <4>
+      }
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The name of the field to contain the generated embeddings for semantic search.
+<2> The identifier of the inference endpoint that generates the embeddings based on the input text.
+<3> The name of the field to contain the original text for lexical search.
+<4> The textual data stored in the `content` field will be copied to `semantic_text` and processed by the {infer} endpoint.
+
+[NOTE]
+====
+If you want to run a search on indices that were populated by web crawlers or connectors, you have to
+<<indices-put-mapping,update the index mappings>> for these indices to
+include the `semantic_text` field. Once the mapping is updated, you'll need to run a full web crawl or a full connector sync. This ensures that all existing
+documents are reprocessed and updated with the new semantic embeddings, enabling hybrid search on the updated data.
+====
+
+[discrete]
+[[semantic-text-hybrid-load-data]]
+==== Load data
+
+In this step, you load the data that you later use to create embeddings from.
+
+Use the `msmarco-passagetest2019-top1000` data set, which is a subset of the MS MARCO Passage Ranking data set. It consists of 200 queries, each accompanied by a list of relevant text passages. All unique passages, along with their IDs, have been extracted from that data set and compiled into a https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file].
+
+Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] in the {ml-app} UI. After your data is analyzed, click **Override settings**. Under **Edit field names**, assign `id` to the first column and `content` to the second. Click **Apply**, then **Import**. Name the index `test-data`, and click **Import**. After the upload is complete, you will see an index named `test-data` with 182,469 documents.
+
+[discrete]
+[[hybrid-search-reindex-data]]
+==== Reindex the data for hybrid search
+
+Reindex the data from the `test-data` index into the `semantic-embeddings` index.
+The data in the `content` field of the source index is copied into the `content` field of the destination index. 
+The `copy_to` parameter set in the index mapping creation ensures that the content is copied into the `semantic_text` field. The data is processed by the {infer} endpoint at ingest time to generate embeddings.
+
+[NOTE]
+====
+This step uses the reindex API to simulate data ingestion. If you are working with data that has already been indexed,
+rather than using the `test-data` set, reindexing is still required to ensure that the data is processed by the {infer} endpoint
+and the necessary embeddings are generated.
+====
+
+[source,console]
+------------------------------------------------------------
+POST _reindex?wait_for_completion=false
+{
+  "source": {
+    "index": "test-data",
+    "size": 10 <1>
+  },
+  "dest": {
+    "index": "semantic-embeddings"
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The default batch size for reindexing is 1000. Reducing size to a smaller
+number makes the update of the reindexing process quicker which enables you to
+follow the progress closely and detect errors early.
+
+The call returns a task ID to monitor the progress:
+
+[source,console]
+------------------------------------------------------------
+GET _tasks/<task_id>
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+Reindexing large datasets can take a long time. You can test this workflow using only a subset of the dataset.
+
+To cancel the reindexing process and generate embeddings for the subset that was reindexed:
+
+[source,console]
+------------------------------------------------------------
+POST _tasks/<task_id>/_cancel
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+[discrete]
+[[hybrid-search-perform-search]]
+==== Perform hybrid search
+
+After reindexing the data into the `semantic-embeddings` index, you can perform hybrid search by using <<rrf,reciprocal rank fusion (RRF)>>. RRF is a technique that merges the rankings from both semantic and lexical queries, giving more weight to results that rank high in either search. This ensures that the final results are balanced and relevant.
+
+[source,console]
+------------------------------------------------------------
+GET semantic-embeddings/_search
+{
+  "retriever": {
+    "rrf": {
+      "retrievers": [
+        {
+          "standard": { <1>
+            "query": {
+              "match": {
+                "content": "How to avoid muscle soreness while running?" <2>
+              }
+            }
+          }
+        },
+        {
+          "standard": { <3>
+            "query": {
+              "semantic": {
+                "field": "semantic_text", <4>
+                "query": "How to avoid muscle soreness while running?"
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The first `standard` retriever represents the traditional lexical search.
+<2> Lexical search is performed on the `content` field using the specified phrase.
+<3> The second `standard` retriever refers to the semantic search.
+<4> The `semantic_text` field is used to perform the semantic search.
+
+
+After performing the hybrid search, the query will return the top 10 documents that match both semantic and lexical search criteria. The results include detailed information about each document:
+
+[source,console-result]
+------------------------------------------------------------
+{
+  "took": 107,
+  "timed_out": false,
+  "_shards": {
+    "total": 1,
+    "successful": 1,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 473,
+      "relation": "eq"
+    },
+    "max_score": null,
+    "hits": [
+      {
+        "_index": "semantic-embeddings",
+        "_id": "wv65epIBEMBRnhfTsOFM", 
+        "_score": 0.032786883,
+        "_rank": 1,
+        "_source": {
+          "semantic_text": {
+            "inference": {
+              "inference_id": "my-elser-endpoint",
+              "model_settings": {
+                "task_type": "sparse_embedding"
+              },
+              "chunks": [
+                {
+                  "text": "What so many out there do not realize is the importance of what you do after you work out. You may have done the majority of the work, but how you treat your body in the minutes and hours after you exercise has a direct effect on muscle soreness, muscle strength and growth, and staying hydrated. Cool Down. After your last exercise, your workout is not over. The first thing you need to do is cool down. Even if running was all that you did, you still should do light cardio for a few minutes. This brings your heart rate down at a slow and steady pace, which helps you avoid feeling sick after a workout.",
+                  "embeddings": {
+                    "exercise": 1.571044,
+                    "after": 1.3603843,
+                    "sick": 1.3281639,
+                    "cool": 1.3227621,
+                    "muscle": 1.2645415,
+                    "sore": 1.2561599,
+                    "cooling": 1.2335974,
+                    "running": 1.1750668,
+                    "hours": 1.1104802,
+                    "out": 1.0991782,
+                    "##io": 1.0794281,
+                    "last": 1.0474665,
+                   (...) 
+                  }
+                }
+              ]
+            }
+          },
+          "id": 8408852,
+          "content": "What so many out there do not realize is the importance of (...)"
+        }
+      }
+    ]
+  }
+}
+------------------------------------------------------------
+// NOTCONSOLE
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 501d645665a02..2ad407b4ae1e4 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -38,7 +38,7 @@ must have the `read` index privilege for the alias's data streams or indices.
 
 Allows you to execute a search query and get back search hits that match the
 query. You can provide search queries using the <<search-api-query-params-q,`q`
-query string parameter>> or <<search-search,request body>>.
+query string parameter>> or <<search-search-api-request-body,request body>>.
 
 [[search-search-api-path-params]]
 ==== {api-path-parms-title}
diff --git a/docs/reference/security/limitations.asciidoc b/docs/reference/security/limitations.asciidoc
index 96af0e01c8075..b1bdd8cbbf5d5 100644
--- a/docs/reference/security/limitations.asciidoc
+++ b/docs/reference/security/limitations.asciidoc
@@ -81,12 +81,13 @@ including the following queries:
 * A search request cannot be profiled if document level security is enabled.
 * The <<search-terms-enum,terms enum API>> does not return terms if document
 level security is enabled.
+* The <<query-dsl-multi-match-query, `multi_match`>> query does not support specifying fields using wildcards.
 
 NOTE: While document-level security prevents users from viewing restricted documents,
 it's still possible to write search requests that return aggregate information about the
 entire index. A user whose access is restricted to specific documents in an index could
 still learn about field names and terms that only exist in inaccessible
-documents, and count how many inaccessible documents contain a given term. 
+documents, and count how many inaccessible documents contain a given term.
 
 [discrete]
 [[alias-limitations]]
diff --git a/docs/reference/snapshot-restore/index.asciidoc b/docs/reference/snapshot-restore/index.asciidoc
index 390f6664391bd..33645034c30a1 100644
--- a/docs/reference/snapshot-restore/index.asciidoc
+++ b/docs/reference/snapshot-restore/index.asciidoc
@@ -48,6 +48,9 @@ Snapshots don't contain or back up:
 * Node configuration files
 * <<security-files,Security configuration files>>
 
+NOTE: When restoring a data stream, if the target cluster does not have an index template that matches the data stream, the data stream will not be able to roll over until a matching index template is created.
+This will affect the lifecycle management of the data stream and interfere with the data stream size and retention.
+
 [discrete]
 [[feature-state]]
 === Feature states
diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc
index 2147ad3c684f3..6c1319c2c71b1 100644
--- a/docs/reference/snapshot-restore/register-repository.asciidoc
+++ b/docs/reference/snapshot-restore/register-repository.asciidoc
@@ -248,10 +248,11 @@ that you have an archive copy of its contents that you can use to recreate the
 repository in its current state at a later date.
 
 You must ensure that {es} does not write to the repository while you are taking
-the backup of its contents. You can do this by unregistering it, or registering
-it with `readonly: true`, on all your clusters. If {es} writes any data to the
-repository during the backup then the contents of the backup may not be
-consistent and it may not be possible to recover any data from it in future.
+the backup of its contents. If {es} writes any data to the repository during
+the backup then the contents of the backup may not be consistent and it may not
+be possible to recover any data from it in future. Prevent writes to the
+repository by unregistering the repository from the cluster which has write
+access to it.
 
 Alternatively, if your repository supports it, you may take an atomic snapshot
 of the underlying filesystem and then take a backup of this filesystem
diff --git a/docs/reference/snapshot-restore/repository-azure.asciidoc b/docs/reference/snapshot-restore/repository-azure.asciidoc
index c361414052e14..0e6e1478cfc55 100644
--- a/docs/reference/snapshot-restore/repository-azure.asciidoc
+++ b/docs/reference/snapshot-restore/repository-azure.asciidoc
@@ -259,6 +259,15 @@ include::repository-shared-settings.asciidoc[]
     `primary_only` or `secondary_only`. Defaults to `primary_only`. Note that if you set it
     to `secondary_only`, it will force `readonly` to true.
 
+`delete_objects_max_size`::
+
+    (integer) Sets the maxmimum batch size, betewen 1 and 256, used for `BlobBatch` requests. Defaults to 256 which is the maximum
+    number supported by the https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch#remarks[Azure blob batch API].
+
+`max_concurrent_batch_deletes`::
+
+    (integer) Sets the maximum number of concurrent batch delete requests that will be submitted for any individual bulk delete with `BlobBatch`. Note that the effective number of concurrent deletes is further limited by the Azure client connection and event loop thread limits. Defaults to 10, minimum is 1, maximum is 100.
+
 [[repository-azure-validation]]
 ==== Repository validation rules
 
diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc
index 71a9fd8b87c96..36f311b1cdd97 100644
--- a/docs/reference/snapshot-restore/repository-s3.asciidoc
+++ b/docs/reference/snapshot-restore/repository-s3.asciidoc
@@ -329,6 +329,25 @@ include::repository-shared-settings.asciidoc[]
     `1000` which is the maximum number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS
     ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up dangling multipart uploads.
 
+`throttled_delete_retry.delay_increment`::
+
+    (<<time-units,time value>>) This value is used as the delay before the first retry and the amount the delay is incremented by on each subsequent retry. Default is 50ms, minimum is 0ms.
+
+`throttled_delete_retry.maximum_delay`::
+
+    (<<time-units,time value>>) This is the upper bound on how long the delays between retries will grow to. Default is 5s, minimum is 0ms.
+
+`throttled_delete_retry.maximum_number_of_retries`::
+
+    (integer) Sets the number times to retry a throttled snapshot deletion. Defaults to `10`, minimum value is `0` which
+    will disable retries altogether. Note that if retries are enabled in the Azure client, each of these retries
+    comprises that many client-level retries.
+
+`get_register_retry_delay`
+
+    (<<time-units,time value>>) Sets the time to wait before trying again if an attempt to read a
+    <<repository-s3-linearizable-registers,linearizable register>> fails. Defaults to `5s`.
+
 NOTE: The option of defining client settings in the repository settings as
 documented below is considered deprecated, and will be removed in a future
 version.
diff --git a/docs/reference/snapshot-restore/repository-source-only.asciidoc b/docs/reference/snapshot-restore/repository-source-only.asciidoc
index 04e53c42aff9d..3c11d6ca6e59c 100644
--- a/docs/reference/snapshot-restore/repository-source-only.asciidoc
+++ b/docs/reference/snapshot-restore/repository-source-only.asciidoc
@@ -27,6 +27,9 @@ As a result, indices adopting synthetic source cannot be restored. When you rest
  * The mapping of the restored index is empty, but the original mapping is available from the types top
    level `meta` element.
 
+ * Information such as document count, deleted document count and store size are not available for such indices
+   since these indices do not contain the relevant data structures to retrieve this information from. Therefore,
+   this information is not shown for such indices in APIs such as the <<cat-indices,cat indices API>>.
 ==================================================
 
 Before registering a source-only repository, use {kib} or the
diff --git a/docs/reference/watcher/how-watcher-works.asciidoc b/docs/reference/watcher/how-watcher-works.asciidoc
index ed6e49b72e9ce..e34d4f799d99b 100644
--- a/docs/reference/watcher/how-watcher-works.asciidoc
+++ b/docs/reference/watcher/how-watcher-works.asciidoc
@@ -146,15 +146,18 @@ add, the more distributed the watches can be executed. If you add or remove
 replicas, all watches need to be reloaded. If a shard is relocated, the
 primary and all replicas of this particular shard will reload.
 
-Because the watches are executed on the node, where the watch shards are, you can create
-dedicated watcher nodes by using shard allocation filtering.
+Because the watches are executed on the node, where the watch shards are, you
+can create dedicated watcher nodes by using shard allocation filtering. To do this
+, configure nodes with a dedicated `node.attr.role: watcher` property.
 
-You could configure nodes with a dedicated `node.attr.role: watcher` property and
-then configure the `.watches` index like this:
+As the `.watches` index is a system index, you can't use the normal `.watcher/_settings`
+endpoint to modify its routing allocation. Instead, you can use the following dedicated
+endpoint to adjust the allocation of the `.watches` shards to the nodes with the
+`watcher` role attribute:
 
 [source,console]
 ------------------------
-PUT .watches/_settings
+PUT _watcher/settings
 {
   "index.routing.allocation.include.role": "watcher"
 }
diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
index 35c26ef10f9ec..d11c4b7fd9c91 100644
--- a/gradle/build.versions.toml
+++ b/gradle/build.versions.toml
@@ -17,7 +17,7 @@ commons-codec = "commons-codec:commons-codec:1.11"
 commmons-io = "commons-io:commons-io:2.2"
 docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5"
 forbiddenApis = "de.thetaphi:forbiddenapis:3.6"
-gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.17.4"
+gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.18.1"
 hamcrest = "org.hamcrest:hamcrest:2.1"
 httpcore = "org.apache.httpcomponents:httpcore:4.4.12"
 httpclient = "org.apache.httpcomponents:httpclient:4.5.14"
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index 443417e6a5b92..5cfe7adb5ea49 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -69,9 +69,9 @@
             <sha256 value="10fe288fd7a2cdaf5175332b73529f9abf8fd54dcfff317d6967c0c35ffb133b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="co.elastic.apm" name="elastic-apm-agent" version="1.44.0">
-         <artifact name="elastic-apm-agent-1.44.0.jar">
-            <sha256 value="169dc01756a960692314c12ed8868baf937399f6eddc2d7e6ede9caad8bec0a2" origin="Generated by Gradle"/>
+      <component group="co.elastic.apm" name="elastic-apm-agent" version="1.52.0">
+         <artifact name="elastic-apm-agent-1.52.0.jar">
+            <sha256 value="ef6c8f75bd6181e717cdd172864441580708c7ee8543175621a3f404f4ba6429" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="co.elastic.logging" name="ecs-logging-core" version="1.2.0">
@@ -144,6 +144,11 @@
             <sha256 value="31915426834400cac854f48441c168d55aa6fc054527f28f1d242a7067affd14" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.azure" name="azure-storage-blob-batch" version="12.23.1">
+         <artifact name="azure-storage-blob-batch-12.23.1.jar">
+            <sha256 value="8c11749c783222873f63f22575aa5ae7ee8f285388183b82d1a18db21f4d2eba" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.azure" name="azure-storage-common" version="12.26.1">
          <artifact name="azure-storage-common-12.26.1.jar">
             <sha256 value="b0297ac1a9017ccd8a1e5cf41fb8d00ff0adbdd06849f6c5aafb3208708264dd" origin="Generated by Gradle"/>
@@ -799,6 +804,11 @@
             <sha256 value="e2b3f8a191b0b401b75c2c4542d3d1719814a4212e6920fae4f2f940678bfd99" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.gradle" name="develocity-gradle-plugin" version="3.18.1">
+         <artifact name="develocity-gradle-plugin-3.18.1.jar">
+            <sha256 value="0adf32afef4ae46a131367e393d18a5d9529750dfd1432287797a64b2c36c7ef" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.h2database" name="h2" version="1.4.197">
          <artifact name="h2-1.4.197.jar">
             <sha256 value="37f5216e14af2772930dff9b8734353f0a80e89ba3f33e065441de6537c5e842" origin="Generated by Gradle"/>
@@ -2819,129 +2829,129 @@
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.12.0">
-         <artifact name="lucene-analysis-common-9.12.0.jar">
-            <sha256 value="8c79d8741f711cc6d7501805b03f7b5f505805d09cab1beb95f0be24b6d27655" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="10.0.0">
+         <artifact name="lucene-analysis-common-10.0.0.jar">
+            <sha256 value="af703beb0898514a1c2d8dd58ac94fa14a6275398288482e1b9cd85f07d56912" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.12.0">
-         <artifact name="lucene-analysis-icu-9.12.0.jar">
-            <sha256 value="52e914cc2d50beec8af0d07f50e5dab803cb9ce8e675ffe43a2aae6102fe6e25" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="10.0.0">
+         <artifact name="lucene-analysis-icu-10.0.0.jar">
+            <sha256 value="0c69ea85db9be3daf6ed692f01e8467ab9b8e8fcc3740169bfbb601e7c5d55e5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.12.0">
-         <artifact name="lucene-analysis-kuromoji-9.12.0.jar">
-            <sha256 value="3da0bd68c68cfe518d0e87ea49af1a9560579bca6b9a7fdf642169cb76dd6887" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="10.0.0">
+         <artifact name="lucene-analysis-kuromoji-10.0.0.jar">
+            <sha256 value="c12cb66f91752c5aa4969ad1a7775d268e5dec3d5e6e78b107fad2a02e897aac" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.12.0">
-         <artifact name="lucene-analysis-morfologik-9.12.0.jar">
-            <sha256 value="920358e33d806b5c3feb3566795714e1f36a616520bd2bd5da1e8b639c1f8549" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="10.0.0">
+         <artifact name="lucene-analysis-morfologik-10.0.0.jar">
+            <sha256 value="99aa16783155c122833f61c84726eedce41b1b0778a861e84d853a3e5397fc6d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.12.0">
-         <artifact name="lucene-analysis-nori-9.12.0.jar">
-            <sha256 value="26cacbe1d75e138ac0ba1e392edb1877c10a422f98524bf51d5d77492272889d" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="10.0.0">
+         <artifact name="lucene-analysis-nori-10.0.0.jar">
+            <sha256 value="5e80fa011ef360f2093fec157959d63af7766c13490f5d32fa3e82260edcf54e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.12.0">
-         <artifact name="lucene-analysis-phonetic-9.12.0.jar">
-            <sha256 value="4bb24414425e04ccd28c835e8dd7f52259185f8f671aea6432460b4b7afa5cfa" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="10.0.0">
+         <artifact name="lucene-analysis-phonetic-10.0.0.jar">
+            <sha256 value="901320e21ce7a15548323be1628df9dc96d83ebca4e0bfa71b3d771bfff5a7bb" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.12.0">
-         <artifact name="lucene-analysis-smartcn-9.12.0.jar">
-            <sha256 value="ffd68804d3387968784b7f52b7c14c7439fa474ffbf1b36d93fa1d2b04101f7a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="10.0.0">
+         <artifact name="lucene-analysis-smartcn-10.0.0.jar">
+            <sha256 value="4d41a52e24f16fb7e9d96371362bf5bba5ae308c2740753fd1ff5e938800c6b3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.12.0">
-         <artifact name="lucene-analysis-stempel-9.12.0.jar">
-            <sha256 value="c4911b5697c0403b694aa84d3f63090067c74ffa8c95cb088d0b80e91166df4a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="10.0.0">
+         <artifact name="lucene-analysis-stempel-10.0.0.jar">
+            <sha256 value="723ca9d879d341283cf2dbe3c6bdf5a13875165aaba6a197dc669152bf0c2047" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.12.0">
-         <artifact name="lucene-backward-codecs-9.12.0.jar">
-            <sha256 value="a42c17f304c9bf1e8411ebdcf91baf2df2d00c7bfa13bca9e41ccf381d78cfba" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="10.0.0">
+         <artifact name="lucene-backward-codecs-10.0.0.jar">
+            <sha256 value="a5439e70d0e0dbc6745354b30b3168eb39f5ea452cc7187a81d30f080f4e87e4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-codecs" version="9.12.0">
-         <artifact name="lucene-codecs-9.12.0.jar">
-            <sha256 value="8173a45b87df23a6dd279916cf16b361c0dcfeff5927121b014d210b19f17555" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-codecs" version="10.0.0">
+         <artifact name="lucene-codecs-10.0.0.jar">
+            <sha256 value="517bd716e6773346b2a8e9f32a72c10b006c63ff6c27733d4808deb41cf01bda" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-core" version="9.12.0">
-         <artifact name="lucene-core-9.12.0.jar">
-            <sha256 value="6c7b774b75cd8f369e246f365a47caa54ae991cae6afa49c7f339e9921ca58a0" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-core" version="10.0.0">
+         <artifact name="lucene-core-10.0.0.jar">
+            <sha256 value="ab187a10018d1e889624ef9e60b6fff0349c6315965b5087bdad0243a735f941" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-expressions" version="9.12.0">
-         <artifact name="lucene-expressions-9.12.0.jar">
-            <sha256 value="9d14919a3d0f07420c1d57a68de52180ae2477d6255d7f693f44515a4fda9c9b" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-expressions" version="10.0.0">
+         <artifact name="lucene-expressions-10.0.0.jar">
+            <sha256 value="4b1b9ffaf2ca65ec57e4af52b76fcbfc975b31d5c836ba343ab471f7d884963e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-facet" version="9.12.0">
-         <artifact name="lucene-facet-9.12.0.jar">
-            <sha256 value="3c07754f0a1f45ba42a637faa8568c18507defa6b77a1ed71af291cd4e7ac603" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-facet" version="10.0.0">
+         <artifact name="lucene-facet-10.0.0.jar">
+            <sha256 value="a760708d9ba1da12803d69a3027e5960b0cf8d6c05f7649716015dfe2a49ea77" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-grouping" version="9.12.0">
-         <artifact name="lucene-grouping-9.12.0.jar">
-            <sha256 value="ffc02d047f4f036c5a5f409baaf8e19abca165deccc3e57e7a422560f4de1219" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-grouping" version="10.0.0">
+         <artifact name="lucene-grouping-10.0.0.jar">
+            <sha256 value="311b875f88cebcdc93a4104d51302eb8c1c278dbcda8a4ae79467a634d4272dc" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-highlighter" version="9.12.0">
-         <artifact name="lucene-highlighter-9.12.0.jar">
-            <sha256 value="5094e1b38f02eb72500a4283ac0f654e2e81943c5359b629567dd7900c935a59" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="10.0.0">
+         <artifact name="lucene-highlighter-10.0.0.jar">
+            <sha256 value="74a2e01e94b040dcaf4b5e989a0a262f2a6b41f41bc2b17e11c9bca3e2b239bb" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-join" version="9.12.0">
-         <artifact name="lucene-join-9.12.0.jar">
-            <sha256 value="c49cb1be843295b2c7158c03755f222416fad9915114bc7f74649d20f4fc43d6" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-join" version="10.0.0">
+         <artifact name="lucene-join-10.0.0.jar">
+            <sha256 value="19b8326e56693824bbb1335d29e378d3288cd97add5c61538b502fb8666dd033" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-memory" version="9.12.0">
-         <artifact name="lucene-memory-9.12.0.jar">
-            <sha256 value="fdbb9c9b3270ce06370d094f1a80dbbbf81b2638b55fa14cc817469092899f01" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-memory" version="10.0.0">
+         <artifact name="lucene-memory-10.0.0.jar">
+            <sha256 value="40eb3a15025bd54ac6715a7f0f68ad3d0130d8a1226398b2d8fd2c7b82538cb5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-misc" version="9.12.0">
-         <artifact name="lucene-misc-9.12.0.jar">
-            <sha256 value="7989d094b4b168ce9ef2949e13f6bfe820829944e50831881be1adeddbe02c04" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-misc" version="10.0.0">
+         <artifact name="lucene-misc-10.0.0.jar">
+            <sha256 value="ef3217c4fe4a294b98a122d6e252eb1d43801201369850345d6d995f1fc7a7b7" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queries" version="9.12.0">
-         <artifact name="lucene-queries-9.12.0.jar">
-            <sha256 value="3b4da6b1673e4edd141c7b86538cc61f6ff3386106f1f5e179c9fe2d30ea2fca" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queries" version="10.0.0">
+         <artifact name="lucene-queries-10.0.0.jar">
+            <sha256 value="2b0372c2d19dac6fb0eb7bc9ccf2dfe7211954d7c82a77c00d15bb5c2ae4196d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queryparser" version="9.12.0">
-         <artifact name="lucene-queryparser-9.12.0.jar">
-            <sha256 value="fc866b7bbfc199436feb24be7dd5c575445342eb283bfe07dd519eab131d3675" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="10.0.0">
+         <artifact name="lucene-queryparser-10.0.0.jar">
+            <sha256 value="29aff63cd8cd8ce6e8db1eeadf4b9634d5688ad1591866e23d6d9059afaf27a0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-sandbox" version="9.12.0">
-         <artifact name="lucene-sandbox-9.12.0.jar">
-            <sha256 value="2262f39a241f435dba61f230e7528205c34f4bf0008faa57f0808476bb782996" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="10.0.0">
+         <artifact name="lucene-sandbox-10.0.0.jar">
+            <sha256 value="58bd8fb5f700b270b9971762269ef209f4c6b9eeadc7ff907b7e15cf6463740a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.12.0">
-         <artifact name="lucene-spatial-extras-9.12.0.jar">
-            <sha256 value="ea41de3b2da573e488033b8b1c2033e17c27ce39957cf6acb8904297ae6e1747" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="10.0.0">
+         <artifact name="lucene-spatial-extras-10.0.0.jar">
+            <sha256 value="e6e7708f51bd5a9ccd37774170a420478011c22e57b262242e207010962e8f01" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.12.0">
-         <artifact name="lucene-spatial3d-9.12.0.jar">
-            <sha256 value="4263e163937024838a8761618b1ce4eba06857adca0c9385d1ac014ec2937de6" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="10.0.0">
+         <artifact name="lucene-spatial3d-10.0.0.jar">
+            <sha256 value="594738cc1b97dfc7dcd856fb10d7ebbeacf9ef14aa658555e4298ae4b6794978" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-suggest" version="9.12.0">
-         <artifact name="lucene-suggest-9.12.0.jar">
-            <sha256 value="1606fb9c7cf57d88b2824916c00280c92e14bcde798a6b78b27e0db8f13227f0" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-suggest" version="10.0.0">
+         <artifact name="lucene-suggest-10.0.0.jar">
+            <sha256 value="662851cf788fd76d846d89c1c9243e956af767402275924bb538468e7aeb35bc" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-test-framework" version="9.12.0">
-         <artifact name="lucene-test-framework-9.12.0.jar">
-            <sha256 value="47d4e1f47ebd26117d47af8fbff471c116c571fd7d7bd6cb199d89f8454e5cf0" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="10.0.0">
+         <artifact name="lucene-test-framework-10.0.0.jar">
+            <sha256 value="8bd7bd94136f5879a9748ab12c2b263a5519d8ce813bfc40d40f5bf086a7bebd" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.maven" name="maven-artifact" version="3.6.1">
@@ -4222,6 +4232,11 @@
             <sha256 value="6e24913b021ffacfe8e7e053d6e0ccc731941148cfa078d4f1ed3d96904530f8" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm-util" version="9.7">
+         <artifact name="asm-util-9.7.jar">
+            <sha256 value="37a6414d36641973f1af104937c95d6d921b2ddb4d612c66c5a9f2b13fc14211" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.reactivestreams" name="reactive-streams" version="1.0.4">
          <artifact name="reactive-streams-1.0.4.jar">
             <sha256 value="f75ca597789b3dac58f61857b9ac2e1034a68fa672db35055a8fb4509e325f28" origin="Generated by Gradle"/>
diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java
index e2aea6b3ebd9f..4ed60b2f5e8b2 100644
--- a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java
+++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 
 import java.util.Optional;
 
@@ -39,7 +39,7 @@ static Optional<VectorScorerFactory> instance() {
     Optional<RandomVectorScorerSupplier> getInt7SQVectorScorerSupplier(
         VectorSimilarityType similarityType,
         IndexInput input,
-        RandomAccessQuantizedByteVectorValues values,
+        QuantizedByteVectorValues values,
         float scoreCorrectionConstant
     );
 
@@ -52,9 +52,5 @@ Optional<RandomVectorScorerSupplier> getInt7SQVectorScorerSupplier(
      * @param queryVector the query vector
      * @return an optional containing the vector scorer, or empty
      */
-    Optional<RandomVectorScorer> getInt7SQVectorScorer(
-        VectorSimilarityFunction sim,
-        RandomAccessQuantizedByteVectorValues values,
-        float[] queryVector
-    );
+    Optional<RandomVectorScorer> getInt7SQVectorScorer(VectorSimilarityFunction sim, QuantizedByteVectorValues values, float[] queryVector);
 }
diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java
index a22d787980252..6248902c32e7a 100644
--- a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java
+++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 
 import java.util.Optional;
 
@@ -25,7 +25,7 @@ final class VectorScorerFactoryImpl implements VectorScorerFactory {
     public Optional<RandomVectorScorerSupplier> getInt7SQVectorScorerSupplier(
         VectorSimilarityType similarityType,
         IndexInput input,
-        RandomAccessQuantizedByteVectorValues values,
+        QuantizedByteVectorValues values,
         float scoreCorrectionConstant
     ) {
         throw new UnsupportedOperationException("should not reach here");
@@ -34,7 +34,7 @@ public Optional<RandomVectorScorerSupplier> getInt7SQVectorScorerSupplier(
     @Override
     public Optional<RandomVectorScorer> getInt7SQVectorScorer(
         VectorSimilarityFunction sim,
-        RandomAccessQuantizedByteVectorValues values,
+        QuantizedByteVectorValues values,
         float[] queryVector
     ) {
         throw new UnsupportedOperationException("should not reach here");
diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java
index a65fe582087d9..a863d9e3448ca 100644
--- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java
+++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java
@@ -15,7 +15,7 @@
 import org.apache.lucene.store.MemorySegmentAccessInput;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.elasticsearch.nativeaccess.NativeAccess;
 import org.elasticsearch.simdvec.internal.Int7SQVectorScorer;
 import org.elasticsearch.simdvec.internal.Int7SQVectorScorerSupplier.DotProductSupplier;
@@ -38,7 +38,7 @@ private VectorScorerFactoryImpl() {}
     public Optional<RandomVectorScorerSupplier> getInt7SQVectorScorerSupplier(
         VectorSimilarityType similarityType,
         IndexInput input,
-        RandomAccessQuantizedByteVectorValues values,
+        QuantizedByteVectorValues values,
         float scoreCorrectionConstant
     ) {
         input = FilterIndexInput.unwrapOnlyTest(input);
@@ -57,7 +57,7 @@ public Optional<RandomVectorScorerSupplier> getInt7SQVectorScorerSupplier(
     @Override
     public Optional<RandomVectorScorer> getInt7SQVectorScorer(
         VectorSimilarityFunction sim,
-        RandomAccessQuantizedByteVectorValues values,
+        QuantizedByteVectorValues values,
         float[] queryVector
     ) {
         return Int7SQVectorScorer.create(sim, values, queryVector);
diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java
index 0b41436ce2242..e02df124ad0f0 100644
--- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java
+++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java
@@ -11,18 +11,14 @@
 
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 
 import java.util.Optional;
 
 public final class Int7SQVectorScorer {
 
     // Unconditionally returns an empty optional on <= JDK 21, since the scorer is only supported on JDK 22+
-    public static Optional<RandomVectorScorer> create(
-        VectorSimilarityFunction sim,
-        RandomAccessQuantizedByteVectorValues values,
-        float[] queryVector
-    ) {
+    public static Optional<RandomVectorScorer> create(VectorSimilarityFunction sim, QuantizedByteVectorValues values, float[] queryVector) {
         return Optional.empty();
     }
 
diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java
index f6d874cd3e728..198e10406056e 100644
--- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java
+++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.store.MemorySegmentAccessInput;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizedVectorSimilarity;
 
 import java.io.IOException;
@@ -31,12 +31,12 @@ public abstract sealed class Int7SQVectorScorerSupplier implements RandomVectorS
     final int maxOrd;
     final float scoreCorrectionConstant;
     final MemorySegmentAccessInput input;
-    final RandomAccessQuantizedByteVectorValues values; // to support ordToDoc/getAcceptOrds
+    final QuantizedByteVectorValues values; // to support ordToDoc/getAcceptOrds
     final ScalarQuantizedVectorSimilarity fallbackScorer;
 
     protected Int7SQVectorScorerSupplier(
         MemorySegmentAccessInput input,
-        RandomAccessQuantizedByteVectorValues values,
+        QuantizedByteVectorValues values,
         float scoreCorrectionConstant,
         ScalarQuantizedVectorSimilarity fallbackScorer
     ) {
@@ -104,11 +104,7 @@ public float score(int node) throws IOException {
 
     public static final class EuclideanSupplier extends Int7SQVectorScorerSupplier {
 
-        public EuclideanSupplier(
-            MemorySegmentAccessInput input,
-            RandomAccessQuantizedByteVectorValues values,
-            float scoreCorrectionConstant
-        ) {
+        public EuclideanSupplier(MemorySegmentAccessInput input, QuantizedByteVectorValues values, float scoreCorrectionConstant) {
             super(input, values, scoreCorrectionConstant, fromVectorSimilarity(EUCLIDEAN, scoreCorrectionConstant, BITS));
         }
 
@@ -127,11 +123,7 @@ public EuclideanSupplier copy() {
 
     public static final class DotProductSupplier extends Int7SQVectorScorerSupplier {
 
-        public DotProductSupplier(
-            MemorySegmentAccessInput input,
-            RandomAccessQuantizedByteVectorValues values,
-            float scoreCorrectionConstant
-        ) {
+        public DotProductSupplier(MemorySegmentAccessInput input, QuantizedByteVectorValues values, float scoreCorrectionConstant) {
             super(input, values, scoreCorrectionConstant, fromVectorSimilarity(DOT_PRODUCT, scoreCorrectionConstant, BITS));
         }
 
@@ -151,11 +143,7 @@ public DotProductSupplier copy() {
 
     public static final class MaxInnerProductSupplier extends Int7SQVectorScorerSupplier {
 
-        public MaxInnerProductSupplier(
-            MemorySegmentAccessInput input,
-            RandomAccessQuantizedByteVectorValues values,
-            float scoreCorrectionConstant
-        ) {
+        public MaxInnerProductSupplier(MemorySegmentAccessInput input, QuantizedByteVectorValues values, float scoreCorrectionConstant) {
             super(input, values, scoreCorrectionConstant, fromVectorSimilarity(MAXIMUM_INNER_PRODUCT, scoreCorrectionConstant, BITS));
         }
 
diff --git a/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java b/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java
index c9659ea1af9a8..3d0e1e71a3744 100644
--- a/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java
+++ b/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java
@@ -15,7 +15,7 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.MemorySegmentAccessInput;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 
 import java.io.IOException;
@@ -35,11 +35,7 @@ public abstract sealed class Int7SQVectorScorer extends RandomVectorScorer.Abstr
     byte[] scratch;
 
     /** Return an optional whose value, if present, is the scorer. Otherwise, an empty optional is returned. */
-    public static Optional<RandomVectorScorer> create(
-        VectorSimilarityFunction sim,
-        RandomAccessQuantizedByteVectorValues values,
-        float[] queryVector
-    ) {
+    public static Optional<RandomVectorScorer> create(VectorSimilarityFunction sim, QuantizedByteVectorValues values, float[] queryVector) {
         checkDimensions(queryVector.length, values.dimension());
         var input = values.getSlice();
         if (input == null) {
@@ -63,12 +59,7 @@ public static Optional<RandomVectorScorer> create(
         };
     }
 
-    Int7SQVectorScorer(
-        MemorySegmentAccessInput input,
-        RandomAccessQuantizedByteVectorValues values,
-        byte[] queryVector,
-        float queryCorrection
-    ) {
+    Int7SQVectorScorer(MemorySegmentAccessInput input, QuantizedByteVectorValues values, byte[] queryVector, float queryCorrection) {
         super(values);
         this.input = input;
         assert queryVector.length == values.getVectorByteLength();
@@ -105,7 +96,7 @@ final void checkOrdinal(int ord) {
     }
 
     public static final class DotProductScorer extends Int7SQVectorScorer {
-        public DotProductScorer(MemorySegmentAccessInput in, RandomAccessQuantizedByteVectorValues values, byte[] query, float correction) {
+        public DotProductScorer(MemorySegmentAccessInput in, QuantizedByteVectorValues values, byte[] query, float correction) {
             super(in, values, query, correction);
         }
 
@@ -122,7 +113,7 @@ public float score(int node) throws IOException {
     }
 
     public static final class EuclideanScorer extends Int7SQVectorScorer {
-        public EuclideanScorer(MemorySegmentAccessInput in, RandomAccessQuantizedByteVectorValues values, byte[] query, float correction) {
+        public EuclideanScorer(MemorySegmentAccessInput in, QuantizedByteVectorValues values, byte[] query, float correction) {
             super(in, values, query, correction);
         }
 
@@ -136,7 +127,7 @@ public float score(int node) throws IOException {
     }
 
     public static final class MaxInnerProductScorer extends Int7SQVectorScorer {
-        public MaxInnerProductScorer(MemorySegmentAccessInput in, RandomAccessQuantizedByteVectorValues values, byte[] query, float corr) {
+        public MaxInnerProductScorer(MemorySegmentAccessInput in, QuantizedByteVectorValues values, byte[] query, float corr) {
             super(in, values, query, corr);
         }
 
diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java
index db57dc936e794..0f967127f6f2c 100644
--- a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java
+++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java
@@ -21,7 +21,7 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 
 import java.io.IOException;
@@ -431,14 +431,13 @@ public Optional<Throwable> call() {
         }
     }
 
-    RandomAccessQuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
+    QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
         var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
         var slice = in.slice("values", 0, in.length());
         return new OffHeapQuantizedByteVectorValues.DenseOffHeapVectorValues(dims, size, sq, false, sim, null, slice);
     }
 
-    RandomVectorScorerSupplier luceneScoreSupplier(RandomAccessQuantizedByteVectorValues values, VectorSimilarityFunction sim)
-        throws IOException {
+    RandomVectorScorerSupplier luceneScoreSupplier(QuantizedByteVectorValues values, VectorSimilarityFunction sim) throws IOException {
         return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorerSupplier(sim, values);
     }
 
diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java
index e0b5875c6c108..4562afa8af693 100644
--- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java
+++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java
@@ -96,6 +96,41 @@ public TokenFilter includeProperty(String name) {
         return filter;
     }
 
+    /**
+     * This is overridden in order to keep empty arrays in nested exclusions - see #109668.
+     * <p>
+     * If we are excluding contents, we only want to exclude based on property name - but empty arrays in themselves do not have a property
+     * name. If the empty array were to be excluded, it should be done by excluding the parent.
+     * <p>
+     * Note though that the expected behavior seems to be ambiguous if contentsFiltered is true - that is, that the filter has pruned all
+     * the contents of a given array, such that we are left with the empty array. The behavior below drops that array, for at the time of
+     * writing, not doing so would cause assertions in JsonXContentFilteringTests to fail, which expect this behavior. Yet it is not obvious
+     * if dropping the empty array in this case is correct. For example, one could expect this sort of behavior:
+     * <ul>
+     *     <li>Document: <pre>{ "myArray": [ { "myField": "myValue" } ]}</pre></li>
+     *     <li>Filter: <pre>{ "exclude": "myArray.myField" }</pre></li>
+     * </ul>
+     * From the user's perspective, this could reasonably yield either of:
+     * <ol>
+     *     <li><pre>{ "myArray": []}</pre></li>
+     *     <li>Removing {@code myArray} entirely.</li>
+     * </ol>
+     */
+    @Override
+    public boolean includeEmptyArray(boolean contentsFiltered) {
+        return inclusive == false && contentsFiltered == false;
+    }
+
+    /**
+     * This is overridden in order to keep empty objects in nested exclusions - see #109668.
+     * <p>
+     * The same logic applies to this as to {@link #includeEmptyArray(boolean)}, only for nested objects instead of nested arrays.
+     */
+    @Override
+    public boolean includeEmptyObject(boolean contentsFiltered) {
+        return inclusive == false && contentsFiltered == false;
+    }
+
     @Override
     protected boolean _includeScalar() {
         return inclusive == false;
diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java
index aa869b1af4f5e..6f0b473b5ba1f 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java
@@ -40,6 +40,7 @@
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.function.LongFunction;
 
 /**
  * A utility to build XContent (ie json).
@@ -107,13 +108,15 @@ public static XContentBuilder builder(XContentType xContentType, Set<String> inc
     private static final Map<Class<?>, Writer> WRITERS;
     private static final Map<Class<?>, HumanReadableTransformer> HUMAN_READABLE_TRANSFORMERS;
     private static final Map<Class<?>, Function<Object, Object>> DATE_TRANSFORMERS;
+    private static final LongFunction<String> UNIX_EPOCH_MILLIS_FORMATTER;
+
     static {
         Map<Class<?>, Writer> writers = new HashMap<>();
         writers.put(Boolean.class, (b, v) -> b.value((Boolean) v));
         writers.put(boolean[].class, (b, v) -> b.values((boolean[]) v));
         writers.put(Byte.class, (b, v) -> b.value((Byte) v));
         writers.put(byte[].class, (b, v) -> b.value((byte[]) v));
-        writers.put(Date.class, XContentBuilder::timeValue);
+        writers.put(Date.class, XContentBuilder::timestampValue);
         writers.put(Double.class, (b, v) -> b.value((Double) v));
         writers.put(double[].class, (b, v) -> b.values((double[]) v));
         writers.put(Float.class, (b, v) -> b.value((Float) v));
@@ -129,8 +132,8 @@ public static XContentBuilder builder(XContentType xContentType, Set<String> inc
         writers.put(Locale.class, (b, v) -> b.value(v.toString()));
         writers.put(Class.class, (b, v) -> b.value(v.toString()));
         writers.put(ZonedDateTime.class, (b, v) -> b.value(v.toString()));
-        writers.put(Calendar.class, XContentBuilder::timeValue);
-        writers.put(GregorianCalendar.class, XContentBuilder::timeValue);
+        writers.put(Calendar.class, XContentBuilder::timestampValue);
+        writers.put(GregorianCalendar.class, XContentBuilder::timestampValue);
         writers.put(BigInteger.class, (b, v) -> b.value((BigInteger) v));
         writers.put(BigDecimal.class, (b, v) -> b.value((BigDecimal) v));
 
@@ -140,6 +143,8 @@ public static XContentBuilder builder(XContentType xContentType, Set<String> inc
         // treat strings as already converted
         dateTransformers.put(String.class, Function.identity());
 
+        LongFunction<String> unixEpochMillisFormatter = Long::toString;
+
         // Load pluggable extensions
         for (XContentBuilderExtension service : ServiceLoader.load(XContentBuilderExtension.class)) {
             Map<Class<?>, Writer> addlWriters = service.getXContentWriters();
@@ -157,11 +162,14 @@ public static XContentBuilder builder(XContentType xContentType, Set<String> inc
             writers.putAll(addlWriters);
             humanReadableTransformer.putAll(addlTransformers);
             dateTransformers.putAll(addlDateTransformers);
+
+            unixEpochMillisFormatter = service::formatUnixEpochMillis;
         }
 
         WRITERS = Map.copyOf(writers);
         HUMAN_READABLE_TRANSFORMERS = Map.copyOf(humanReadableTransformer);
         DATE_TRANSFORMERS = Map.copyOf(dateTransformers);
+        UNIX_EPOCH_MILLIS_FORMATTER = unixEpochMillisFormatter;
     }
 
     @FunctionalInterface
@@ -797,52 +805,53 @@ public XContentBuilder utf8Value(byte[] bytes, int offset, int length) throws IO
     }
 
     ////////////////////////////////////////////////////////////////////////////
-    // Date
+    // Timestamps
     //////////////////////////////////
 
     /**
-     * Write a time-based field and value, if the passed timeValue is null a
-     * null value is written, otherwise a date transformers lookup is performed.
-
-     * @throws IllegalArgumentException if there is no transformers for the type of object
+     * Write a field with a timestamp value: if the passed timestamp is null then writes null, otherwise looks up the date transformer
+     * for the type of {@code timestamp} and uses it to format the value.
+     *
+     * @throws IllegalArgumentException if there is no transformer for the given value type
      */
-    public XContentBuilder timeField(String name, Object timeValue) throws IOException {
-        return field(name).timeValue(timeValue);
+    public XContentBuilder timestampField(String name, Object timestamp) throws IOException {
+        return field(name).timestampValue(timestamp);
     }
 
     /**
-     * If the {@code humanReadable} flag is set, writes both a formatted and
-     * unformatted version of the time value using the date transformer for the
-     * {@link Long} class.
+     * Writes a field containing the raw number of milliseconds since the unix epoch, and also if the {@code humanReadable} flag is set,
+     * writes a formatted representation of this value using the UNIX_EPOCH_MILLIS_FORMATTER.
      */
-    public XContentBuilder timeField(String name, String readableName, long value) throws IOException {
-        assert name.equals(readableName) == false : "expected raw and readable field names to differ, but they were both: " + name;
+    public XContentBuilder timestampFieldsFromUnixEpochMillis(String rawFieldName, String humanReadableFieldName, long unixEpochMillis)
+        throws IOException {
+        assert rawFieldName.equals(humanReadableFieldName) == false
+            : "expected raw and readable field names to differ, but they were both: " + rawFieldName;
         if (humanReadable) {
-            Function<Object, Object> longTransformer = DATE_TRANSFORMERS.get(Long.class);
-            if (longTransformer == null) {
-                throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type Long");
-            }
-            field(readableName).value(longTransformer.apply(value));
+            field(humanReadableFieldName, UNIX_EPOCH_MILLIS_FORMATTER.apply(unixEpochMillis));
         }
-        field(name, value);
+        field(rawFieldName, unixEpochMillis);
         return this;
     }
 
     /**
-     * Write a time-based value, if the value is null a null value is written,
-     * otherwise a date transformers lookup is performed.
-
-     * @throws IllegalArgumentException if there is no transformers for the type of object
+     * Write a timestamp value: if the passed timestamp is null then writes null, otherwise looks up the date transformer for the type of
+     * {@code timestamp} and uses it to format the value.
+     *
+     * @throws IllegalArgumentException if there is no transformer for the given value type
      */
-    public XContentBuilder timeValue(Object timeValue) throws IOException {
-        if (timeValue == null) {
+    public XContentBuilder timestampValue(Object timestamp) throws IOException {
+        if (timestamp == null) {
             return nullValue();
         } else {
-            Function<Object, Object> transformer = DATE_TRANSFORMERS.get(timeValue.getClass());
+            Function<Object, Object> transformer = DATE_TRANSFORMERS.get(timestamp.getClass());
             if (transformer == null) {
-                throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type " + timeValue.getClass());
+                final var exception = new IllegalArgumentException(
+                    "cannot write timestamp value xcontent for value of unknown type " + timestamp.getClass()
+                );
+                assert false : exception;
+                throw exception;
             }
-            return value(transformer.apply(timeValue));
+            return value(transformer.apply(timestamp));
         }
     }
 
diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java
index 1e48667079cfc..4e3b442e7d473 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java
@@ -68,4 +68,9 @@ public interface XContentBuilderExtension {
      * </pre>
      */
     Map<Class<?>, Function<Object, Object>> getDateTransformers();
+
+    /**
+     * Used to format a {@code long} representing the number of milliseconds since the Unix Epoch.
+     */
+    String formatUnixEpochMillis(long unixEpochMillis);
 }
diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle
index 1b3aac13b3608..f558ce8b9cfdb 100644
--- a/modules/aggregations/build.gradle
+++ b/modules/aggregations/build.gradle
@@ -48,4 +48,5 @@ dependencies {
 
 tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed")
+  task.skipTest("aggregations/time_series/Configure with no synthetic source", "temporary until backport")
 })
diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java
index 3e66bf0edf394..e911bf1a41198 100644
--- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java
+++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java
@@ -65,17 +65,17 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     private static IndexRequestBuilder indexDoc(String idx, ZonedDateTime date, int value) throws Exception {
-        return prepareIndex(idx).setSource(jsonBuilder().startObject().timeField("date", date).field("value", value).endObject());
+        return prepareIndex(idx).setSource(jsonBuilder().startObject().timestampField("date", date).field("value", value).endObject());
     }
 
     private IndexRequestBuilder indexDoc(int month, int day, int value) throws Exception {
         return prepareIndex("idx").setSource(
             jsonBuilder().startObject()
                 .field("value", value)
-                .timeField("date", date(month, day))
+                .timestampField("date", date(month, day))
                 .startArray("dates")
-                .timeValue(date(month, day))
-                .timeValue(date(month + 1, day + 1))
+                .timestampValue(date(month, day))
+                .timestampValue(date(month + 1, day + 1))
                 .endArray()
                 .endObject()
         );
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
index 1703d4908a753..acab855e17df6 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
@@ -291,23 +291,6 @@ setup:
                     sum:
                       sum:
                         field: val
----
-"Configure with no synthetic source":
-  - requires:
-      cluster_features: ["gte_v8.15.0"]
-      reason: "Error message changed in 8.15.0"
-
-  - do:
-      catch: '/Indices with with index mode \[time_series\] only support synthetic source/'
-      indices.create:
-        index: tsdb_error
-        body:
-          settings:
-            mode: time_series
-            routing_path: [key]
-          mappings:
-            _source:
-              enabled: false
 
 ---
 "Number for keyword routing field":
diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle
index b16c6eaaaa1d1..f4f7e787d2b7b 100644
--- a/modules/analysis-common/build.gradle
+++ b/modules/analysis-common/build.gradle
@@ -33,3 +33,7 @@ dependencies {
 artifacts {
   restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
 }
+
+tasks.named("yamlRestCompatTestTransform").configure { task ->
+  task.replaceValueInMatch("tokens.0.token", "absenț", "romanian")
+}
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyRomanianStemmer.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyRomanianStemmer.java
new file mode 100644
index 0000000000000..0eb8d916307ae
--- /dev/null
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyRomanianStemmer.java
@@ -0,0 +1,741 @@
+/*
+ * @notice
+ * Generated by Snowball 2.0.0 - https://snowballstem.org/
+ *
+ * Modifications copyright (C) 2024 Elasticsearch B.V.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.tartarus.snowball.Among;
+
+/**
+* This class implements the stemming algorithm defined by a snowball script.
+* NOTE: This is the RomanianStemmer used in Lucene 9 and should only be used for backwards compatibility
+*/
+@SuppressWarnings("checkstyle:DescendantToken")
+class LegacyRomanianStemmer extends org.tartarus.snowball.SnowballStemmer {
+
+    private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
+
+    private static final Among a_0[] = { new Among("", -1, 3), new Among("I", 0, 1), new Among("U", 0, 2) };
+
+    private static final Among a_1[] = {
+        new Among("ea", -1, 3),
+        new Among("a\u0163ia", -1, 7),
+        new Among("aua", -1, 2),
+        new Among("iua", -1, 4),
+        new Among("a\u0163ie", -1, 7),
+        new Among("ele", -1, 3),
+        new Among("ile", -1, 5),
+        new Among("iile", 6, 4),
+        new Among("iei", -1, 4),
+        new Among("atei", -1, 6),
+        new Among("ii", -1, 4),
+        new Among("ului", -1, 1),
+        new Among("ul", -1, 1),
+        new Among("elor", -1, 3),
+        new Among("ilor", -1, 4),
+        new Among("iilor", 14, 4) };
+
+    private static final Among a_2[] = {
+        new Among("icala", -1, 4),
+        new Among("iciva", -1, 4),
+        new Among("ativa", -1, 5),
+        new Among("itiva", -1, 6),
+        new Among("icale", -1, 4),
+        new Among("a\u0163iune", -1, 5),
+        new Among("i\u0163iune", -1, 6),
+        new Among("atoare", -1, 5),
+        new Among("itoare", -1, 6),
+        new Among("\u0103toare", -1, 5),
+        new Among("icitate", -1, 4),
+        new Among("abilitate", -1, 1),
+        new Among("ibilitate", -1, 2),
+        new Among("ivitate", -1, 3),
+        new Among("icive", -1, 4),
+        new Among("ative", -1, 5),
+        new Among("itive", -1, 6),
+        new Among("icali", -1, 4),
+        new Among("atori", -1, 5),
+        new Among("icatori", 18, 4),
+        new Among("itori", -1, 6),
+        new Among("\u0103tori", -1, 5),
+        new Among("icitati", -1, 4),
+        new Among("abilitati", -1, 1),
+        new Among("ivitati", -1, 3),
+        new Among("icivi", -1, 4),
+        new Among("ativi", -1, 5),
+        new Among("itivi", -1, 6),
+        new Among("icit\u0103i", -1, 4),
+        new Among("abilit\u0103i", -1, 1),
+        new Among("ivit\u0103i", -1, 3),
+        new Among("icit\u0103\u0163i", -1, 4),
+        new Among("abilit\u0103\u0163i", -1, 1),
+        new Among("ivit\u0103\u0163i", -1, 3),
+        new Among("ical", -1, 4),
+        new Among("ator", -1, 5),
+        new Among("icator", 35, 4),
+        new Among("itor", -1, 6),
+        new Among("\u0103tor", -1, 5),
+        new Among("iciv", -1, 4),
+        new Among("ativ", -1, 5),
+        new Among("itiv", -1, 6),
+        new Among("ical\u0103", -1, 4),
+        new Among("iciv\u0103", -1, 4),
+        new Among("ativ\u0103", -1, 5),
+        new Among("itiv\u0103", -1, 6) };
+
+    private static final Among a_3[] = {
+        new Among("ica", -1, 1),
+        new Among("abila", -1, 1),
+        new Among("ibila", -1, 1),
+        new Among("oasa", -1, 1),
+        new Among("ata", -1, 1),
+        new Among("ita", -1, 1),
+        new Among("anta", -1, 1),
+        new Among("ista", -1, 3),
+        new Among("uta", -1, 1),
+        new Among("iva", -1, 1),
+        new Among("ic", -1, 1),
+        new Among("ice", -1, 1),
+        new Among("abile", -1, 1),
+        new Among("ibile", -1, 1),
+        new Among("isme", -1, 3),
+        new Among("iune", -1, 2),
+        new Among("oase", -1, 1),
+        new Among("ate", -1, 1),
+        new Among("itate", 17, 1),
+        new Among("ite", -1, 1),
+        new Among("ante", -1, 1),
+        new Among("iste", -1, 3),
+        new Among("ute", -1, 1),
+        new Among("ive", -1, 1),
+        new Among("ici", -1, 1),
+        new Among("abili", -1, 1),
+        new Among("ibili", -1, 1),
+        new Among("iuni", -1, 2),
+        new Among("atori", -1, 1),
+        new Among("osi", -1, 1),
+        new Among("ati", -1, 1),
+        new Among("itati", 30, 1),
+        new Among("iti", -1, 1),
+        new Among("anti", -1, 1),
+        new Among("isti", -1, 3),
+        new Among("uti", -1, 1),
+        new Among("i\u015Fti", -1, 3),
+        new Among("ivi", -1, 1),
+        new Among("it\u0103i", -1, 1),
+        new Among("o\u015Fi", -1, 1),
+        new Among("it\u0103\u0163i", -1, 1),
+        new Among("abil", -1, 1),
+        new Among("ibil", -1, 1),
+        new Among("ism", -1, 3),
+        new Among("ator", -1, 1),
+        new Among("os", -1, 1),
+        new Among("at", -1, 1),
+        new Among("it", -1, 1),
+        new Among("ant", -1, 1),
+        new Among("ist", -1, 3),
+        new Among("ut", -1, 1),
+        new Among("iv", -1, 1),
+        new Among("ic\u0103", -1, 1),
+        new Among("abil\u0103", -1, 1),
+        new Among("ibil\u0103", -1, 1),
+        new Among("oas\u0103", -1, 1),
+        new Among("at\u0103", -1, 1),
+        new Among("it\u0103", -1, 1),
+        new Among("ant\u0103", -1, 1),
+        new Among("ist\u0103", -1, 3),
+        new Among("ut\u0103", -1, 1),
+        new Among("iv\u0103", -1, 1) };
+
+    private static final Among a_4[] = {
+        new Among("ea", -1, 1),
+        new Among("ia", -1, 1),
+        new Among("esc", -1, 1),
+        new Among("\u0103sc", -1, 1),
+        new Among("ind", -1, 1),
+        new Among("\u00E2nd", -1, 1),
+        new Among("are", -1, 1),
+        new Among("ere", -1, 1),
+        new Among("ire", -1, 1),
+        new Among("\u00E2re", -1, 1),
+        new Among("se", -1, 2),
+        new Among("ase", 10, 1),
+        new Among("sese", 10, 2),
+        new Among("ise", 10, 1),
+        new Among("use", 10, 1),
+        new Among("\u00E2se", 10, 1),
+        new Among("e\u015Fte", -1, 1),
+        new Among("\u0103\u015Fte", -1, 1),
+        new Among("eze", -1, 1),
+        new Among("ai", -1, 1),
+        new Among("eai", 19, 1),
+        new Among("iai", 19, 1),
+        new Among("sei", -1, 2),
+        new Among("e\u015Fti", -1, 1),
+        new Among("\u0103\u015Fti", -1, 1),
+        new Among("ui", -1, 1),
+        new Among("ezi", -1, 1),
+        new Among("\u00E2i", -1, 1),
+        new Among("a\u015Fi", -1, 1),
+        new Among("se\u015Fi", -1, 2),
+        new Among("ase\u015Fi", 29, 1),
+        new Among("sese\u015Fi", 29, 2),
+        new Among("ise\u015Fi", 29, 1),
+        new Among("use\u015Fi", 29, 1),
+        new Among("\u00E2se\u015Fi", 29, 1),
+        new Among("i\u015Fi", -1, 1),
+        new Among("u\u015Fi", -1, 1),
+        new Among("\u00E2\u015Fi", -1, 1),
+        new Among("a\u0163i", -1, 2),
+        new Among("ea\u0163i", 38, 1),
+        new Among("ia\u0163i", 38, 1),
+        new Among("e\u0163i", -1, 2),
+        new Among("i\u0163i", -1, 2),
+        new Among("\u00E2\u0163i", -1, 2),
+        new Among("ar\u0103\u0163i", -1, 1),
+        new Among("ser\u0103\u0163i", -1, 2),
+        new Among("aser\u0103\u0163i", 45, 1),
+        new Among("seser\u0103\u0163i", 45, 2),
+        new Among("iser\u0103\u0163i", 45, 1),
+        new Among("user\u0103\u0163i", 45, 1),
+        new Among("\u00E2ser\u0103\u0163i", 45, 1),
+        new Among("ir\u0103\u0163i", -1, 1),
+        new Among("ur\u0103\u0163i", -1, 1),
+        new Among("\u00E2r\u0103\u0163i", -1, 1),
+        new Among("am", -1, 1),
+        new Among("eam", 54, 1),
+        new Among("iam", 54, 1),
+        new Among("em", -1, 2),
+        new Among("asem", 57, 1),
+        new Among("sesem", 57, 2),
+        new Among("isem", 57, 1),
+        new Among("usem", 57, 1),
+        new Among("\u00E2sem", 57, 1),
+        new Among("im", -1, 2),
+        new Among("\u00E2m", -1, 2),
+        new Among("\u0103m", -1, 2),
+        new Among("ar\u0103m", 65, 1),
+        new Among("ser\u0103m", 65, 2),
+        new Among("aser\u0103m", 67, 1),
+        new Among("seser\u0103m", 67, 2),
+        new Among("iser\u0103m", 67, 1),
+        new Among("user\u0103m", 67, 1),
+        new Among("\u00E2ser\u0103m", 67, 1),
+        new Among("ir\u0103m", 65, 1),
+        new Among("ur\u0103m", 65, 1),
+        new Among("\u00E2r\u0103m", 65, 1),
+        new Among("au", -1, 1),
+        new Among("eau", 76, 1),
+        new Among("iau", 76, 1),
+        new Among("indu", -1, 1),
+        new Among("\u00E2ndu", -1, 1),
+        new Among("ez", -1, 1),
+        new Among("easc\u0103", -1, 1),
+        new Among("ar\u0103", -1, 1),
+        new Among("ser\u0103", -1, 2),
+        new Among("aser\u0103", 84, 1),
+        new Among("seser\u0103", 84, 2),
+        new Among("iser\u0103", 84, 1),
+        new Among("user\u0103", 84, 1),
+        new Among("\u00E2ser\u0103", 84, 1),
+        new Among("ir\u0103", -1, 1),
+        new Among("ur\u0103", -1, 1),
+        new Among("\u00E2r\u0103", -1, 1),
+        new Among("eaz\u0103", -1, 1) };
+
+    private static final Among a_5[] = {
+        new Among("a", -1, 1),
+        new Among("e", -1, 1),
+        new Among("ie", 1, 1),
+        new Among("i", -1, 1),
+        new Among("\u0103", -1, 1) };
+
+    private static final char g_v[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4 };
+
+    private boolean B_standard_suffix_removed;
+    private int I_p2;
+    private int I_p1;
+    private int I_pV;
+
+    private boolean r_prelude() {
+        while (true) {
+            int v_1 = cursor;
+            lab0: {
+                golab1: while (true) {
+                    int v_2 = cursor;
+                    lab2: {
+                        if (!(in_grouping(g_v, 97, 259))) {
+                            break lab2;
+                        }
+                        bra = cursor;
+                        lab3: {
+                            int v_3 = cursor;
+                            lab4: {
+                                if (!(eq_s("u"))) {
+                                    break lab4;
+                                }
+                                ket = cursor;
+                                if (!(in_grouping(g_v, 97, 259))) {
+                                    break lab4;
+                                }
+                                slice_from("U");
+                                break lab3;
+                            }
+                            cursor = v_3;
+                            if (!(eq_s("i"))) {
+                                break lab2;
+                            }
+                            ket = cursor;
+                            if (!(in_grouping(g_v, 97, 259))) {
+                                break lab2;
+                            }
+                            slice_from("I");
+                        }
+                        cursor = v_2;
+                        break golab1;
+                    }
+                    cursor = v_2;
+                    if (cursor >= limit) {
+                        break lab0;
+                    }
+                    cursor++;
+                }
+                continue;
+            }
+            cursor = v_1;
+            break;
+        }
+        return true;
+    }
+
+    private boolean r_mark_regions() {
+        I_pV = limit;
+        I_p1 = limit;
+        I_p2 = limit;
+        int v_1 = cursor;
+        lab0: {
+            lab1: {
+                int v_2 = cursor;
+                lab2: {
+                    if (!(in_grouping(g_v, 97, 259))) {
+                        break lab2;
+                    }
+                    lab3: {
+                        int v_3 = cursor;
+                        lab4: {
+                            if (!(out_grouping(g_v, 97, 259))) {
+                                break lab4;
+                            }
+                            golab5: while (true) {
+                                lab6: {
+                                    if (!(in_grouping(g_v, 97, 259))) {
+                                        break lab6;
+                                    }
+                                    break golab5;
+                                }
+                                if (cursor >= limit) {
+                                    break lab4;
+                                }
+                                cursor++;
+                            }
+                            break lab3;
+                        }
+                        cursor = v_3;
+                        if (!(in_grouping(g_v, 97, 259))) {
+                            break lab2;
+                        }
+                        golab7: while (true) {
+                            lab8: {
+                                if (!(out_grouping(g_v, 97, 259))) {
+                                    break lab8;
+                                }
+                                break golab7;
+                            }
+                            if (cursor >= limit) {
+                                break lab2;
+                            }
+                            cursor++;
+                        }
+                    }
+                    break lab1;
+                }
+                cursor = v_2;
+                if (!(out_grouping(g_v, 97, 259))) {
+                    break lab0;
+                }
+                lab9: {
+                    int v_6 = cursor;
+                    lab10: {
+                        if (!(out_grouping(g_v, 97, 259))) {
+                            break lab10;
+                        }
+                        golab11: while (true) {
+                            lab12: {
+                                if (!(in_grouping(g_v, 97, 259))) {
+                                    break lab12;
+                                }
+                                break golab11;
+                            }
+                            if (cursor >= limit) {
+                                break lab10;
+                            }
+                            cursor++;
+                        }
+                        break lab9;
+                    }
+                    cursor = v_6;
+                    if (!(in_grouping(g_v, 97, 259))) {
+                        break lab0;
+                    }
+                    if (cursor >= limit) {
+                        break lab0;
+                    }
+                    cursor++;
+                }
+            }
+            I_pV = cursor;
+        }
+        cursor = v_1;
+        int v_8 = cursor;
+        lab13: {
+            golab14: while (true) {
+                lab15: {
+                    if (!(in_grouping(g_v, 97, 259))) {
+                        break lab15;
+                    }
+                    break golab14;
+                }
+                if (cursor >= limit) {
+                    break lab13;
+                }
+                cursor++;
+            }
+            golab16: while (true) {
+                lab17: {
+                    if (!(out_grouping(g_v, 97, 259))) {
+                        break lab17;
+                    }
+                    break golab16;
+                }
+                if (cursor >= limit) {
+                    break lab13;
+                }
+                cursor++;
+            }
+            I_p1 = cursor;
+            golab18: while (true) {
+                lab19: {
+                    if (!(in_grouping(g_v, 97, 259))) {
+                        break lab19;
+                    }
+                    break golab18;
+                }
+                if (cursor >= limit) {
+                    break lab13;
+                }
+                cursor++;
+            }
+            golab20: while (true) {
+                lab21: {
+                    if (!(out_grouping(g_v, 97, 259))) {
+                        break lab21;
+                    }
+                    break golab20;
+                }
+                if (cursor >= limit) {
+                    break lab13;
+                }
+                cursor++;
+            }
+            I_p2 = cursor;
+        }
+        cursor = v_8;
+        return true;
+    }
+
+    private boolean r_postlude() {
+        int among_var;
+        while (true) {
+            int v_1 = cursor;
+            lab0: {
+                bra = cursor;
+                among_var = find_among(a_0);
+                if (among_var == 0) {
+                    break lab0;
+                }
+                ket = cursor;
+                switch (among_var) {
+                    case 1:
+                        slice_from("i");
+                        break;
+                    case 2:
+                        slice_from("u");
+                        break;
+                    case 3:
+                        if (cursor >= limit) {
+                            break lab0;
+                        }
+                        cursor++;
+                        break;
+                }
+                continue;
+            }
+            cursor = v_1;
+            break;
+        }
+        return true;
+    }
+
+    private boolean r_RV() {
+        if (!(I_pV <= cursor)) {
+            return false;
+        }
+        return true;
+    }
+
+    private boolean r_R1() {
+        if (!(I_p1 <= cursor)) {
+            return false;
+        }
+        return true;
+    }
+
+    private boolean r_R2() {
+        if (!(I_p2 <= cursor)) {
+            return false;
+        }
+        return true;
+    }
+
+    private boolean r_step_0() {
+        int among_var;
+        ket = cursor;
+        among_var = find_among_b(a_1);
+        if (among_var == 0) {
+            return false;
+        }
+        bra = cursor;
+        if (!r_R1()) {
+            return false;
+        }
+        switch (among_var) {
+            case 1:
+                slice_del();
+                break;
+            case 2:
+                slice_from("a");
+                break;
+            case 3:
+                slice_from("e");
+                break;
+            case 4:
+                slice_from("i");
+                break;
+            case 5: {
+                int v_1 = limit - cursor;
+                lab0: {
+                    if (!(eq_s_b("ab"))) {
+                        break lab0;
+                    }
+                    return false;
+                }
+                cursor = limit - v_1;
+            }
+                slice_from("i");
+                break;
+            case 6:
+                slice_from("at");
+                break;
+            case 7:
+                slice_from("a\u0163i");
+                break;
+        }
+        return true;
+    }
+
+    private boolean r_combo_suffix() {
+        int among_var;
+        int v_1 = limit - cursor;
+        ket = cursor;
+        among_var = find_among_b(a_2);
+        if (among_var == 0) {
+            return false;
+        }
+        bra = cursor;
+        if (!r_R1()) {
+            return false;
+        }
+        switch (among_var) {
+            case 1:
+                slice_from("abil");
+                break;
+            case 2:
+                slice_from("ibil");
+                break;
+            case 3:
+                slice_from("iv");
+                break;
+            case 4:
+                slice_from("ic");
+                break;
+            case 5:
+                slice_from("at");
+                break;
+            case 6:
+                slice_from("it");
+                break;
+        }
+        B_standard_suffix_removed = true;
+        cursor = limit - v_1;
+        return true;
+    }
+
+    private boolean r_standard_suffix() {
+        int among_var;
+        B_standard_suffix_removed = false;
+        while (true) {
+            int v_1 = limit - cursor;
+            lab0: {
+                if (!r_combo_suffix()) {
+                    break lab0;
+                }
+                continue;
+            }
+            cursor = limit - v_1;
+            break;
+        }
+        ket = cursor;
+        among_var = find_among_b(a_3);
+        if (among_var == 0) {
+            return false;
+        }
+        bra = cursor;
+        if (!r_R2()) {
+            return false;
+        }
+        switch (among_var) {
+            case 1:
+                slice_del();
+                break;
+            case 2:
+                if (!(eq_s_b("\u0163"))) {
+                    return false;
+                }
+                bra = cursor;
+                slice_from("t");
+                break;
+            case 3:
+                slice_from("ist");
+                break;
+        }
+        B_standard_suffix_removed = true;
+        return true;
+    }
+
+    private boolean r_verb_suffix() {
+        int among_var;
+        if (cursor < I_pV) {
+            return false;
+        }
+        int v_2 = limit_backward;
+        limit_backward = I_pV;
+        ket = cursor;
+        among_var = find_among_b(a_4);
+        if (among_var == 0) {
+            limit_backward = v_2;
+            return false;
+        }
+        bra = cursor;
+        switch (among_var) {
+            case 1:
+                lab0: {
+                    int v_3 = limit - cursor;
+                    lab1: {
+                        if (!(out_grouping_b(g_v, 97, 259))) {
+                            break lab1;
+                        }
+                        break lab0;
+                    }
+                    cursor = limit - v_3;
+                    if (!(eq_s_b("u"))) {
+                        limit_backward = v_2;
+                        return false;
+                    }
+                }
+                slice_del();
+                break;
+            case 2:
+                slice_del();
+                break;
+        }
+        limit_backward = v_2;
+        return true;
+    }
+
+    private boolean r_vowel_suffix() {
+        ket = cursor;
+        if (find_among_b(a_5) == 0) {
+            return false;
+        }
+        bra = cursor;
+        if (!r_RV()) {
+            return false;
+        }
+        slice_del();
+        return true;
+    }
+
+    @Override
+    public boolean stem() {
+        int v_1 = cursor;
+        r_prelude();
+        cursor = v_1;
+        r_mark_regions();
+        limit_backward = cursor;
+        cursor = limit;
+        int v_3 = limit - cursor;
+        r_step_0();
+        cursor = limit - v_3;
+        int v_4 = limit - cursor;
+        r_standard_suffix();
+        cursor = limit - v_4;
+        int v_5 = limit - cursor;
+        lab0: {
+            lab1: {
+                int v_6 = limit - cursor;
+                lab2: {
+                    if (!(B_standard_suffix_removed)) {
+                        break lab2;
+                    }
+                    break lab1;
+                }
+                cursor = limit - v_6;
+                if (!r_verb_suffix()) {
+                    break lab0;
+                }
+            }
+        }
+        cursor = limit - v_5;
+        int v_7 = limit - cursor;
+        r_vowel_suffix();
+        cursor = limit - v_7;
+        cursor = limit_backward;
+        int v_8 = cursor;
+        r_postlude();
+        cursor = v_8;
+        return true;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o instanceof LegacyRomanianStemmer;
+    }
+
+    @Override
+    public int hashCode() {
+        return LegacyRomanianStemmer.class.getName().hashCode();
+    }
+}
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java
index 9ea3a9fa4eee9..917a45188123c 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java
@@ -9,24 +9,72 @@
 
 package org.elasticsearch.analysis.common;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
+import org.apache.lucene.analysis.core.DecimalDigitFilter;
 import org.apache.lucene.analysis.fa.PersianAnalyzer;
+import org.apache.lucene.analysis.fa.PersianCharFilter;
+import org.apache.lucene.analysis.fa.PersianNormalizationFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider;
 import org.elasticsearch.index.analysis.Analysis;
 
-public class PersianAnalyzerProvider extends AbstractIndexAnalyzerProvider<PersianAnalyzer> {
+import java.io.Reader;
 
-    private final PersianAnalyzer analyzer;
+public class PersianAnalyzerProvider extends AbstractIndexAnalyzerProvider<StopwordAnalyzerBase> {
+
+    private final StopwordAnalyzerBase analyzer;
 
     PersianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(name, settings);
-        analyzer = new PersianAnalyzer(Analysis.parseStopWords(env, settings, PersianAnalyzer.getDefaultStopSet()));
+        if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) {
+            // since Lucene 10 this analyzer contains stemming by default
+            analyzer = new PersianAnalyzer(Analysis.parseStopWords(env, settings, PersianAnalyzer.getDefaultStopSet()));
+        } else {
+            // for older index versions we need the old analyzer behaviour without stemming
+            analyzer = new StopwordAnalyzerBase(Analysis.parseStopWords(env, settings, PersianAnalyzer.getDefaultStopSet())) {
+
+                protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
+                    final Tokenizer source = new StandardTokenizer();
+                    TokenStream result = new LowerCaseFilter(source);
+                    result = new DecimalDigitFilter(result);
+                    result = new ArabicNormalizationFilter(result);
+                    /* additional persian-specific normalization */
+                    result = new PersianNormalizationFilter(result);
+                    /*
+                     * the order here is important: the stopword list is normalized with the
+                     * above!
+                     */
+                    return new TokenStreamComponents(source, new StopFilter(result, stopwords));
+                }
+
+                protected TokenStream normalize(String fieldName, TokenStream in) {
+                    TokenStream result = new LowerCaseFilter(in);
+                    result = new DecimalDigitFilter(result);
+                    result = new ArabicNormalizationFilter(result);
+                    /* additional persian-specific normalization */
+                    result = new PersianNormalizationFilter(result);
+                    return result;
+                }
+
+                protected Reader initReader(String fieldName, Reader reader) {
+                    return new PersianCharFilter(reader);
+                }
+            };
+        }
     }
 
     @Override
-    public PersianAnalyzer get() {
+    public StopwordAnalyzerBase get() {
         return this.analyzer;
     }
 }
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java
index cf33a38abd634..6c28df83a6d36 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java
@@ -9,28 +9,60 @@
 
 package org.elasticsearch.analysis.common;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
 import org.apache.lucene.analysis.ro.RomanianAnalyzer;
+import org.apache.lucene.analysis.snowball.SnowballFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider;
 import org.elasticsearch.index.analysis.Analysis;
 
-public class RomanianAnalyzerProvider extends AbstractIndexAnalyzerProvider<RomanianAnalyzer> {
+public class RomanianAnalyzerProvider extends AbstractIndexAnalyzerProvider<StopwordAnalyzerBase> {
 
-    private final RomanianAnalyzer analyzer;
+    private final StopwordAnalyzerBase analyzer;
 
     RomanianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(name, settings);
-        analyzer = new RomanianAnalyzer(
-            Analysis.parseStopWords(env, settings, RomanianAnalyzer.getDefaultStopSet()),
-            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
-        );
+        CharArraySet stopwords = Analysis.parseStopWords(env, settings, RomanianAnalyzer.getDefaultStopSet());
+        CharArraySet stemExclusionSet = Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET);
+        if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) {
+            // since Lucene 10, this analyzer a modern unicode form and normalizes cedilla forms to forms with commas
+            analyzer = new RomanianAnalyzer(stopwords, stemExclusionSet);
+        } else {
+            // for older index versions we need the old behaviour without normalization
+            analyzer = new StopwordAnalyzerBase(Analysis.parseStopWords(env, settings, RomanianAnalyzer.getDefaultStopSet())) {
+
+                protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
+                    final Tokenizer source = new StandardTokenizer();
+                    TokenStream result = new LowerCaseFilter(source);
+                    result = new StopFilter(result, stopwords);
+                    if (stemExclusionSet.isEmpty() == false) {
+                        result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+                    }
+                    result = new SnowballFilter(result, new LegacyRomanianStemmer());
+                    return new TokenStreamComponents(source, result);
+                }
+
+                protected TokenStream normalize(String fieldName, TokenStream in) {
+                    return new LowerCaseFilter(in);
+                }
+            };
+
+        }
     }
 
     @Override
-    public RomanianAnalyzer get() {
+    public StopwordAnalyzerBase get() {
         return this.analyzer;
     }
 }
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
index 1c71c64311517..7548c8ad2b88b 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.analysis.common;
 
+import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.ar.ArabicStemFilter;
 import org.apache.lucene.analysis.bg.BulgarianStemFilter;
@@ -38,8 +39,9 @@
 import org.apache.lucene.analysis.lv.LatvianStemFilter;
 import org.apache.lucene.analysis.miscellaneous.EmptyTokenStream;
 import org.apache.lucene.analysis.no.NorwegianLightStemFilter;
-import org.apache.lucene.analysis.no.NorwegianLightStemmer;
+import org.apache.lucene.analysis.no.NorwegianLightStemFilterFactory;
 import org.apache.lucene.analysis.no.NorwegianMinimalStemFilter;
+import org.apache.lucene.analysis.no.NorwegianMinimalStemFilterFactory;
 import org.apache.lucene.analysis.pt.PortugueseLightStemFilter;
 import org.apache.lucene.analysis.pt.PortugueseMinimalStemFilter;
 import org.apache.lucene.analysis.pt.PortugueseStemFilter;
@@ -62,14 +64,11 @@
 import org.tartarus.snowball.ext.EstonianStemmer;
 import org.tartarus.snowball.ext.FinnishStemmer;
 import org.tartarus.snowball.ext.FrenchStemmer;
-import org.tartarus.snowball.ext.German2Stemmer;
 import org.tartarus.snowball.ext.GermanStemmer;
 import org.tartarus.snowball.ext.HungarianStemmer;
 import org.tartarus.snowball.ext.IrishStemmer;
 import org.tartarus.snowball.ext.ItalianStemmer;
-import org.tartarus.snowball.ext.KpStemmer;
 import org.tartarus.snowball.ext.LithuanianStemmer;
-import org.tartarus.snowball.ext.LovinsStemmer;
 import org.tartarus.snowball.ext.NorwegianStemmer;
 import org.tartarus.snowball.ext.PortugueseStemmer;
 import org.tartarus.snowball.ext.RomanianStemmer;
@@ -80,6 +79,7 @@
 import org.tartarus.snowball.ext.TurkishStemmer;
 
 import java.io.IOException;
+import java.util.Collections;
 
 public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {
 
@@ -87,27 +87,15 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private static final TokenStream EMPTY_TOKEN_STREAM = new EmptyTokenStream();
 
-    private String language;
+    private final String language;
+
+    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(StemmerTokenFilterFactory.class);
 
     StemmerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
         super(name, settings);
         this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter")));
         // check that we have a valid language by trying to create a TokenStream
         create(EMPTY_TOKEN_STREAM).close();
-        if ("lovins".equalsIgnoreCase(language)) {
-            deprecationLogger.critical(
-                DeprecationCategory.ANALYSIS,
-                "lovins_deprecation",
-                "The [lovins] stemmer is deprecated and will be removed in a future version."
-            );
-        }
-        if ("dutch_kp".equalsIgnoreCase(language) || "dutchKp".equalsIgnoreCase(language) || "kp".equalsIgnoreCase(language)) {
-            deprecationLogger.critical(
-                DeprecationCategory.ANALYSIS,
-                "dutch_kp_deprecation",
-                "The [dutch_kp] stemmer is deprecated and will be removed in a future version."
-            );
-        }
     }
 
     @Override
@@ -135,8 +123,17 @@ public TokenStream create(TokenStream tokenStream) {
         } else if ("dutch".equalsIgnoreCase(language)) {
             return new SnowballFilter(tokenStream, new DutchStemmer());
         } else if ("dutch_kp".equalsIgnoreCase(language) || "dutchKp".equalsIgnoreCase(language) || "kp".equalsIgnoreCase(language)) {
-            return new SnowballFilter(tokenStream, new KpStemmer());
-
+            deprecationLogger.critical(
+                DeprecationCategory.ANALYSIS,
+                "dutch_kp_deprecation",
+                "The [dutch_kp] stemmer is deprecated and will be removed in a future version."
+            );
+            return new TokenFilter(tokenStream) {
+                @Override
+                public boolean incrementToken() {
+                    return false;
+                }
+            };
             // English stemmers
         } else if ("english".equalsIgnoreCase(language)) {
             return new PorterStemFilter(tokenStream);
@@ -145,7 +142,17 @@ public TokenStream create(TokenStream tokenStream) {
             || "kstem".equalsIgnoreCase(language)) {
                 return new KStemFilter(tokenStream);
             } else if ("lovins".equalsIgnoreCase(language)) {
-                return new SnowballFilter(tokenStream, new LovinsStemmer());
+                deprecationLogger.critical(
+                    DeprecationCategory.ANALYSIS,
+                    "lovins_deprecation",
+                    "The [lovins] stemmer is deprecated and will be removed in a future version."
+                );
+                return new TokenFilter(tokenStream) {
+                    @Override
+                    public boolean incrementToken() {
+                        return false;
+                    }
+                };
             } else if ("porter".equalsIgnoreCase(language)) {
                 return new PorterStemFilter(tokenStream);
             } else if ("porter2".equalsIgnoreCase(language)) {
@@ -185,7 +192,13 @@ public TokenStream create(TokenStream tokenStream) {
             } else if ("german".equalsIgnoreCase(language)) {
                 return new SnowballFilter(tokenStream, new GermanStemmer());
             } else if ("german2".equalsIgnoreCase(language)) {
-                return new SnowballFilter(tokenStream, new German2Stemmer());
+                DEPRECATION_LOGGER.critical(
+                    DeprecationCategory.ANALYSIS,
+                    "german2_stemmer_deprecation",
+                    "The 'german2' stemmer has been deprecated and folded into the 'german' Stemmer. "
+                        + "Replace all usages of 'german2' with 'german'."
+                );
+                return new SnowballFilter(tokenStream, new GermanStemmer());
             } else if ("light_german".equalsIgnoreCase(language) || "lightGerman".equalsIgnoreCase(language)) {
                 return new GermanLightStemFilter(tokenStream);
             } else if ("minimal_german".equalsIgnoreCase(language) || "minimalGerman".equalsIgnoreCase(language)) {
@@ -231,10 +244,13 @@ public TokenStream create(TokenStream tokenStream) {
 
                 // Norwegian (Nynorsk) stemmers
             } else if ("light_nynorsk".equalsIgnoreCase(language) || "lightNynorsk".equalsIgnoreCase(language)) {
-                return new NorwegianLightStemFilter(tokenStream, NorwegianLightStemmer.NYNORSK);
+                NorwegianLightStemFilterFactory factory = new NorwegianLightStemFilterFactory(Collections.singletonMap("variant", "nn"));
+                return factory.create(tokenStream);
             } else if ("minimal_nynorsk".equalsIgnoreCase(language) || "minimalNynorsk".equalsIgnoreCase(language)) {
-                return new NorwegianMinimalStemFilter(tokenStream, NorwegianLightStemmer.NYNORSK);
-
+                NorwegianMinimalStemFilterFactory factory = new NorwegianMinimalStemFilterFactory(
+                    Collections.singletonMap("variant", "nn")
+                );
+                return factory.create(tokenStream);
                 // Persian stemmers
             } else if ("persian".equalsIgnoreCase(language)) {
                 return new PersianStemFilter(tokenStream);
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java
index b406fa8335779..0d936666e92cd 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java
@@ -278,7 +278,7 @@ public void testPhrasePrefix() throws IOException {
                 boolQuery().should(matchPhrasePrefixQuery("field1", "test")).should(matchPhrasePrefixQuery("field1", "bro"))
             ).highlighter(highlight().field("field1").order("score").preTags("<x>").postTags("</x>")),
             resp -> {
-                assertThat(resp.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(resp.getHits().getTotalHits().value(), equalTo(2L));
                 for (int i = 0; i < 2; i++) {
                     assertHighlight(
                         resp,
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java
new file mode 100644
index 0000000000000..7b962538c2a10
--- /dev/null
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.index.IndexVersionUtils;
+
+import java.io.IOException;
+
+import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo;
+
+/**
+ * Tests Persian Analyzer factory and behavioural changes with Lucene 10
+ */
+public class PersianAnalyzerProviderTests extends ESTokenStreamTestCase {
+
+    public void testPersianAnalyzerPostLucene10() throws IOException {
+        IndexVersion postLucene10Version = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.UPGRADE_TO_LUCENE_10_0_0,
+            IndexVersion.current()
+        );
+        Settings settings = ESTestCase.indexSettings(1, 1)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, postLucene10Version)
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+        Environment environment = new Environment(settings, null);
+
+        PersianAnalyzerProvider persianAnalyzerProvider = new PersianAnalyzerProvider(
+            idxSettings,
+            environment,
+            "my-analyzer",
+            Settings.EMPTY
+        );
+        Analyzer analyzer = persianAnalyzerProvider.get();
+        assertAnalyzesTo(analyzer, "من کتاب های زیادی خوانده ام", new String[] { "كتاب", "زياد", "خوانده" });
+    }
+
+    public void testPersianAnalyzerPreLucene10() throws IOException {
+        IndexVersion preLucene10Version = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersionUtils.getFirstVersion(),
+            IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
+        );
+        Settings settings = ESTestCase.indexSettings(1, 1)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, preLucene10Version)
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+        Environment environment = new Environment(settings, null);
+
+        PersianAnalyzerProvider persianAnalyzerProvider = new PersianAnalyzerProvider(
+            idxSettings,
+            environment,
+            "my-analyzer",
+            Settings.EMPTY
+        );
+        Analyzer analyzer = persianAnalyzerProvider.get();
+        assertAnalyzesTo(analyzer, "من کتاب های زیادی خوانده ام", new String[] { "كتاب", "زيادي", "خوانده" });
+    }
+}
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java
new file mode 100644
index 0000000000000..1af44bc71f35d
--- /dev/null
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.index.IndexVersionUtils;
+
+import java.io.IOException;
+
+import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo;
+
+/**
+ * Verifies the behavior of Romanian analyzer.
+ */
+public class RomanianAnalyzerTests extends ESTokenStreamTestCase {
+
+    public void testRomanianAnalyzerPostLucene10() throws IOException {
+        IndexVersion postLucene10Version = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.UPGRADE_TO_LUCENE_10_0_0,
+            IndexVersion.current()
+        );
+        Settings settings = ESTestCase.indexSettings(1, 1)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, postLucene10Version)
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+        Environment environment = new Environment(settings, null);
+
+        RomanianAnalyzerProvider romanianAnalyzerProvider = new RomanianAnalyzerProvider(
+            idxSettings,
+            environment,
+            "my-analyzer",
+            Settings.EMPTY
+        );
+        Analyzer analyzer = romanianAnalyzerProvider.get();
+        assertAnalyzesTo(analyzer, "absenţa", new String[] { "absenț" });
+        assertAnalyzesTo(analyzer, "cunoştinţă", new String[] { "cunoștinț" });
+    }
+
+    public void testRomanianAnalyzerPreLucene10() throws IOException {
+        IndexVersion preLucene10Version = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersionUtils.getFirstVersion(),
+            IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
+        );
+        Settings settings = ESTestCase.indexSettings(1, 1)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, preLucene10Version)
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+        Environment environment = new Environment(settings, null);
+
+        RomanianAnalyzerProvider romanianAnalyzerProvider = new RomanianAnalyzerProvider(
+            idxSettings,
+            environment,
+            "my-analyzer",
+            Settings.EMPTY
+        );
+        Analyzer analyzer = romanianAnalyzerProvider.get();
+        assertAnalyzesTo(analyzer, "absenţa", new String[] { "absenţ" });
+        assertAnalyzesTo(analyzer, "cunoştinţă", new String[] { "cunoştinţ" });
+    }
+}
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
index 8f3d52f0174c6..bb06c221873b5 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
@@ -8,6 +8,7 @@
  */
 package org.elasticsearch.analysis.common;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
@@ -16,6 +17,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalysisTestsHelper;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -103,6 +105,42 @@ public void testMultipleLanguagesThrowsException() throws IOException {
         assertEquals("Invalid stemmer class specified: [english, light_english]", e.getMessage());
     }
 
+    public void testGermanAndGerman2Stemmer() throws IOException {
+        IndexVersion v = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.UPGRADE_TO_LUCENE_10_0_0, IndexVersion.current());
+        Analyzer analyzer = createGermanStemmer("german", v);
+        assertAnalyzesTo(analyzer, "Buecher Bücher", new String[] { "Buch", "Buch" });
+
+        analyzer = createGermanStemmer("german2", v);
+        assertAnalyzesTo(analyzer, "Buecher Bücher", new String[] { "Buch", "Buch" });
+        assertWarnings(
+            "The 'german2' stemmer has been deprecated and folded into the 'german' Stemmer. "
+                + "Replace all usages of 'german2' with 'german'."
+        );
+    }
+
+    private static Analyzer createGermanStemmer(String variant, IndexVersion v) throws IOException {
+
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.my_german.type", "stemmer")
+            .put("index.analysis.filter.my_german.language", variant)
+            .put("index.analysis.analyzer.my_german.tokenizer", "whitespace")
+            .put("index.analysis.analyzer.my_german.filter", "my_german")
+            .put(SETTING_VERSION_CREATED, v)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN);
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_german");
+        assertThat(tokenFilter, instanceOf(StemmerTokenFilterFactory.class));
+        Tokenizer tokenizer = new WhitespaceTokenizer();
+        tokenizer.setReader(new StringReader("Buecher oder Bücher"));
+        TokenStream create = tokenFilter.create(tokenizer);
+        assertThat(create, instanceOf(SnowballFilter.class));
+        IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
+        NamedAnalyzer analyzer = indexAnalyzers.get("my_german");
+        return analyzer;
+    }
+
     public void testKpDeprecation() throws IOException {
         IndexVersion v = IndexVersionUtils.randomVersion(random());
         Settings settings = Settings.builder()
diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/20_analyzers.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/20_analyzers.yml
index c03bdb3111050..8930e485aa249 100644
--- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/20_analyzers.yml
+++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/20_analyzers.yml
@@ -901,6 +901,31 @@
     - length: { tokens: 1 }
     - match:  { tokens.0.token: خورد }
 
+---
+"persian stemming":
+    - requires:
+        cluster_features: ["lucene_10_upgrade"]
+        reason: "test requires persian analyzer stemming capabilities that come with Lucene 10"
+
+    - do:
+        indices.create:
+          index: test
+          body:
+            settings:
+              analysis:
+                analyzer:
+                  my_analyzer:
+                    type: persian
+
+    - do:
+        indices.analyze:
+          index: test
+          body:
+            text: كتابها
+            analyzer: my_analyzer
+    - length: { tokens: 1 }
+    - match: { tokens.0.token: كتاب }
+
 ---
 "portuguese":
     - do:
@@ -948,7 +973,7 @@
             text:     absenţa
             analyzer: romanian
     - length: { tokens: 1 }
-    - match:  { tokens.0.token: absenţ }
+    - match:  { tokens.0.token: absenț }
 
     - do:
         indices.analyze:
@@ -957,7 +982,7 @@
             text:     absenţa
             analyzer: my_analyzer
     - length: { tokens: 1 }
-    - match:  { tokens.0.token: absenţ }
+    - match:  { tokens.0.token: absenț }
 
 ---
 "russian":
diff --git a/modules/apm/build.gradle b/modules/apm/build.gradle
index 4c822e44da6f6..b510e2403e933 100644
--- a/modules/apm/build.gradle
+++ b/modules/apm/build.gradle
@@ -19,7 +19,7 @@ dependencies {
   implementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-context:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-semconv:${otelSemconvVersion}"
-  runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.44.0"
+  runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.52.0"
 }
 
 tasks.named("dependencyLicenses").configure {
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java
index 8f1c0cf515e14..cb74d62137815 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java
@@ -24,7 +24,6 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.Build;
@@ -440,13 +439,13 @@ private static CharacterRunAutomaton buildAutomaton(List<String> includePatterns
             ? includeAutomaton
             : Operations.minus(includeAutomaton, excludeAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
 
-        return new CharacterRunAutomaton(MinimizationOperations.minimize(finalAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT));
+        return new CharacterRunAutomaton(Operations.determinize(finalAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT));
     }
 
     private static Automaton patternsToAutomaton(List<String> patterns) {
         final List<Automaton> automata = patterns.stream().map(s -> {
             final String regex = s.replace(".", "\\.").replace("*", ".*");
-            return new RegExp(regex).toAutomaton();
+            return new RegExp(regex, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton();
         }).toList();
         if (automata.isEmpty()) {
             return null;
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
index 8e7ecfa49f144..777ddc28fefdc 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
@@ -1706,7 +1706,7 @@ public void testSegmentsSortedOnTimestampDesc() throws Exception {
         assertResponse(
             prepareSearch("metrics-foo").addFetchField(new FieldAndFormat(DEFAULT_TIMESTAMP_FIELD, "epoch_millis")).setSize(totalDocs),
             resp -> {
-                assertEquals(totalDocs, resp.getHits().getTotalHits().value);
+                assertEquals(totalDocs, resp.getHits().getTotalHits().value());
                 SearchHit[] hits = resp.getHits().getHits();
                 assertEquals(totalDocs, hits.length);
 
@@ -2027,7 +2027,7 @@ static void indexDocs(String dataStream, int numDocs) {
 
     static void verifyDocs(String dataStream, long expectedNumHits, List<String> expectedIndices) {
         assertResponse(prepareSearch(dataStream).setSize((int) expectedNumHits), resp -> {
-            assertThat(resp.getHits().getTotalHits().value, equalTo(expectedNumHits));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(expectedNumHits));
             Arrays.stream(resp.getHits().getHits()).forEach(hit -> assertTrue(expectedIndices.contains(hit.getIndex())));
         });
     }
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
index 36fb02dcff0d8..286ad68896797 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -45,6 +46,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.RestoreInfo;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
@@ -62,7 +64,9 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoWarningHeaderOnResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertWarningHeaderOnResponse;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -80,6 +84,8 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
     private static final Map<String, Integer> DOCUMENT_SOURCE = Collections.singletonMap("@timestamp", 123);
     public static final String REPO = "repo";
     public static final String SNAPSHOT = "snap";
+    public static final String TEMPLATE_1_ID = "t1";
+    public static final String TEMPLATE_2_ID = "t2";
     private Client client;
 
     private String dsBackingIndexName;
@@ -103,8 +109,8 @@ public void setup() throws Exception {
         Path location = randomRepoPath();
         createRepository(REPO, "fs", location);
 
-        DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds"));
-        DataStreamIT.putComposableIndexTemplate("t2", """
+        DataStreamIT.putComposableIndexTemplate(TEMPLATE_1_ID, List.of("ds", "other-ds"));
+        DataStreamIT.putComposableIndexTemplate(TEMPLATE_2_ID, """
             {
                 "properties": {
                   "@timestamp": {
@@ -132,9 +138,7 @@ public void setup() throws Exception {
         // Initialize the failure store.
         RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null);
         rolloverRequest.setIndicesOptions(
-            IndicesOptions.builder(rolloverRequest.indicesOptions())
-                .failureStoreOptions(b -> b.includeRegularIndices(false).includeFailureIndices(true))
-                .build()
+            IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
         );
         response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get();
         assertTrue(response.isAcknowledged());
@@ -1335,4 +1339,149 @@ public void testRestoreDataStreamAliasWithConflictingIndicesAlias() throws Excep
         );
         assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)"));
     }
+
+    public void testWarningHeaderOnRestoreWithoutTemplates() throws Exception {
+        String datastreamName = "ds";
+
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName)
+            .setIncludeGlobalState(false)
+            .get();
+
+        RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+        SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
+        assertEquals(RestStatus.OK, status);
+
+        assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
+
+        assertAcked(
+            client.execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds")
+            )
+        );
+
+        assertAcked(
+            client.execute(
+                TransportDeleteComposableIndexTemplateAction.TYPE,
+                new TransportDeleteComposableIndexTemplateAction.Request(TEMPLATE_1_ID)
+            ).get()
+        );
+
+        RestoreSnapshotRequestBuilder request = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName);
+
+        assertWarningHeaderOnResponse(
+            client,
+            request,
+            "Snapshot ["
+                + snapshotId
+                + "] contains data stream ["
+                + datastreamName
+                + "] but custer does not have a matching index "
+                + "template. This will cause rollover to fail until a matching index template is created"
+        );
+
+    }
+
+    public void testWarningHeaderAbsentOnRestoreWithTemplates() throws Exception {
+        String datastreamName = "ds";
+
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName)
+            .setIncludeGlobalState(false)
+            .get();
+
+        RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+        SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
+        assertEquals(RestStatus.OK, status);
+
+        assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
+
+        assertAcked(
+            client.execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds", "with-fs")
+            )
+        );
+
+        RestoreSnapshotRequestBuilder request = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName);
+
+        assertNoWarningHeaderOnResponse(
+            client,
+            request,
+            "but custer does not have a matching index template. This will cause rollover to fail until a matching index "
+                + "template is created"
+        );
+
+    }
+
+    /**
+     * This test is muted as it's awaiting the same fix as {@link #testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState()}
+     */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
+    public void testWarningHeaderOnRestoreTemplateFromSnapshot() throws Exception {
+        String datastreamName = "ds";
+
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName)
+            .setIncludeGlobalState(true)
+            .get();
+
+        RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+        SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
+        assertEquals(RestStatus.OK, status);
+
+        assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
+
+        assertAcked(
+            client.execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds")
+            )
+        );
+
+        assertAcked(
+            client.execute(
+                TransportDeleteComposableIndexTemplateAction.TYPE,
+                new TransportDeleteComposableIndexTemplateAction.Request(TEMPLATE_1_ID)
+            ).get()
+        );
+
+        RestoreSnapshotRequestBuilder request = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setRestoreGlobalState(true)
+            .setIndices(datastreamName);
+
+        assertNoWarningHeaderOnResponse(
+            client,
+            request,
+            "Snapshot ["
+                + snapshotId
+                + "] contains data stream ["
+                + datastreamName
+                + "] but custer does not have a matching index "
+                + "template. This will cause rollover to fail until a matching index template is created"
+        );
+
+    }
+
 }
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
index b5d06dc33e035..96def04069e24 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
@@ -195,9 +195,7 @@ public void testRejectionFromFailureStore() throws IOException {
         // Initialize failure store.
         var rolloverRequest = new RolloverRequest(dataStream, null);
         rolloverRequest.setIndicesOptions(
-            IndicesOptions.builder(rolloverRequest.indicesOptions())
-                .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
-                .build()
+            IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
         );
         var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet();
         var failureStoreIndex = rolloverResponse.getNewIndex();
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java
index 83d34571a1597..00dfd5c65b126 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java
@@ -18,17 +18,19 @@
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.disruption.IntermittentLongGCDisruption;
-import org.elasticsearch.test.disruption.SingleNodeDisruption;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -43,7 +45,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testRolloverIsExecutedOnce() throws ExecutionException, InterruptedException {
-        String masterNode = internalCluster().startMasterOnlyNode();
+        internalCluster().startMasterOnlyNode();
         internalCluster().startDataOnlyNodes(3);
         ensureStableCluster(4);
 
@@ -51,7 +53,7 @@ public void testRolloverIsExecutedOnce() throws ExecutionException, InterruptedE
         createDataStream(dataStreamName);
 
         // Mark it to lazy rollover
-        new RolloverRequestBuilder(client()).setRolloverTarget(dataStreamName).lazy(true).execute().get();
+        safeGet(new RolloverRequestBuilder(client()).setRolloverTarget(dataStreamName).lazy(true).execute());
 
         // Verify that the data stream is marked for rollover and that it has currently one index
         DataStream dataStream = getDataStream(dataStreamName);
@@ -59,9 +61,22 @@ public void testRolloverIsExecutedOnce() throws ExecutionException, InterruptedE
         assertThat(dataStream.getBackingIndices().getIndices().size(), equalTo(1));
 
         // Introduce a disruption to the master node that should delay the rollover execution
-        SingleNodeDisruption masterNodeDisruption = new IntermittentLongGCDisruption(random(), masterNode, 100, 200, 30000, 60000);
-        internalCluster().setDisruptionScheme(masterNodeDisruption);
-        masterNodeDisruption.startDisrupting();
+        final var barrier = new CyclicBarrier(2);
+        internalCluster().getCurrentMasterNodeInstance(ClusterService.class)
+            .submitUnbatchedStateUpdateTask("block", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    safeAwait(barrier);
+                    safeAwait(barrier);
+                    return currentState;
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    fail(e);
+                }
+            });
+        safeAwait(barrier);
 
         // Start indexing operations
         int docs = randomIntBetween(5, 10);
@@ -84,10 +99,10 @@ public void onFailure(Exception e) {
         }
 
         // End the disruption so that all pending tasks will complete
-        masterNodeDisruption.stopDisrupting();
+        safeAwait(barrier);
 
         // Wait for all the indexing requests to be processed successfully
-        countDownLatch.await();
+        safeAwait(countDownLatch);
 
         // Verify that the rollover has happened once
         dataStream = getDataStream(dataStreamName);
@@ -96,10 +111,12 @@ public void onFailure(Exception e) {
     }
 
     private DataStream getDataStream(String dataStreamName) {
-        return client().execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName })
-        ).actionGet().getDataStreams().get(0).getDataStream();
+        return safeGet(
+            client().execute(
+                GetDataStreamAction.INSTANCE,
+                new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName })
+            )
+        ).getDataStreams().get(0).getDataStream();
     }
 
     private void createDataStream(String dataStreamName) throws InterruptedException, ExecutionException {
@@ -111,10 +128,9 @@ private void createDataStream(String dataStreamName) throws InterruptedException
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
                 .build()
         );
-        final AcknowledgedResponse putComposableTemplateResponse = client().execute(
-            TransportPutComposableIndexTemplateAction.TYPE,
-            putComposableTemplateRequest
-        ).actionGet();
+        final AcknowledgedResponse putComposableTemplateResponse = safeGet(
+            client().execute(TransportPutComposableIndexTemplateAction.TYPE, putComposableTemplateRequest)
+        );
         assertThat(putComposableTemplateResponse.isAcknowledged(), is(true));
 
         final CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
@@ -122,8 +138,9 @@ private void createDataStream(String dataStreamName) throws InterruptedException
             TEST_REQUEST_TIMEOUT,
             dataStreamName
         );
-        final AcknowledgedResponse createDataStreamResponse = client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest)
-            .get();
+        final AcknowledgedResponse createDataStreamResponse = safeGet(
+            client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest)
+        );
         assertThat(createDataStreamResponse.isAcknowledged(), is(true));
     }
 }
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
index 686e253d1d173..29ec326548f2b 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -170,7 +171,7 @@ public void testTimeRanges() throws Exception {
             var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE);
             time = randomBoolean() ? endTime : endTime.plusSeconds(randomIntBetween(1, 99));
             indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON);
-            expectThrows(IllegalArgumentException.class, () -> client().index(indexRequest).actionGet());
+            expectThrows(IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class, () -> client().index(indexRequest).actionGet());
         }
 
         // Fetch UpdateTimeSeriesRangeService and increment time range of latest backing index:
@@ -545,7 +546,7 @@ public void testTrimId() throws Exception {
         var searchRequest = new SearchRequest(dataStreamName);
         searchRequest.source().trackTotalHits(true);
         assertResponse(client().search(searchRequest), searchResponse -> {
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) numBulkRequests * numDocsPerBulk));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numBulkRequests * numDocsPerBulk));
             String id = searchResponse.getHits().getHits()[0].getId();
             assertThat(id, notNullValue());
 
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java
new file mode 100644
index 0000000000000..980cc32a12c68
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.datastreams;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * This should be a yaml test, but in order to write one we would need to expose the new APIs in the rest-api-spec.
+ * We do not want to do that until the feature flag is removed. For this reason, we temporarily, test the new APIs here.
+ * Please convert this to a yaml test when the feature flag is removed.
+ */
+public class DataStreamOptionsIT extends DisabledSecurityDataStreamTestCase {
+
+    private static final String DATA_STREAM_NAME = "failure-data-stream";
+
+    @SuppressWarnings("unchecked")
+    @Before
+    public void setup() throws IOException {
+        Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/ds-template");
+        putComposableIndexTemplateRequest.setJsonEntity("""
+            {
+              "index_patterns": ["failure-data-stream"],
+              "template": {
+                "settings": {
+                  "number_of_replicas": 0
+                }
+              },
+              "data_stream": {
+                "failure_store": true
+              }
+            }
+            """);
+        assertOK(client().performRequest(putComposableIndexTemplateRequest));
+
+        assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME)));
+        // Initialize the failure store.
+        assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store")));
+        ensureGreen(DATA_STREAM_NAME);
+
+        final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME));
+        List<Object> dataStreams = (List<Object>) entityAsMap(dataStreamResponse).get("data_streams");
+        assertThat(dataStreams.size(), is(1));
+        Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+        assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME));
+        List<String> backingIndices = getIndices(dataStream);
+        assertThat(backingIndices.size(), is(1));
+        List<String> failureStore = getFailureStore(dataStream);
+        assertThat(failureStore.size(), is(1));
+    }
+
+    public void testEnableDisableFailureStore() throws IOException {
+        {
+            assertAcknowledged(client().performRequest(new Request("DELETE", "/_data_stream/" + DATA_STREAM_NAME + "/_options")));
+            assertFailureStore(false, 1);
+            assertDataStreamOptions(null);
+        }
+        {
+            Request enableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options");
+            enableRequest.setJsonEntity("""
+                {
+                  "failure_store": {
+                    "enabled": true
+                  }
+                }""");
+            assertAcknowledged(client().performRequest(enableRequest));
+            assertFailureStore(true, 1);
+            assertDataStreamOptions(true);
+        }
+
+        {
+            Request disableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options");
+            disableRequest.setJsonEntity("""
+                {
+                  "failure_store": {
+                    "enabled": false
+                  }
+                }""");
+            assertAcknowledged(client().performRequest(disableRequest));
+            assertFailureStore(false, 1);
+            assertDataStreamOptions(false);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertFailureStore(boolean failureStoreEnabled, int failureStoreSize) throws IOException {
+        final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME));
+        List<Object> dataStreams = (List<Object>) entityAsMap(dataStreamResponse).get("data_streams");
+        assertThat(dataStreams.size(), is(1));
+        Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+        assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME));
+        assertThat(dataStream.containsKey("failure_store"), is(true));
+        // Ensure the failure store is set to the provided value
+        assertThat(((Map<String, Object>) dataStream.get("failure_store")).get("enabled"), equalTo(failureStoreEnabled));
+        // And the failure indices preserved
+        List<String> failureStore = getFailureStore(dataStream);
+        assertThat(failureStore.size(), is(failureStoreSize));
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertDataStreamOptions(Boolean failureStoreEnabled) throws IOException {
+        final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME + "/_options"));
+        List<Object> dataStreams = (List<Object>) entityAsMap(dataStreamResponse).get("data_streams");
+        assertThat(dataStreams.size(), is(1));
+        Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+        assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME));
+        Map<String, Map<String, Object>> options = (Map<String, Map<String, Object>>) dataStream.get("options");
+        if (failureStoreEnabled == null) {
+            assertThat(options, nullValue());
+        } else {
+            assertThat(options.containsKey("failure_store"), is(true));
+            assertThat(options.get("failure_store").get("enabled"), equalTo(failureStoreEnabled));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private List<String> getFailureStore(Map<String, Object> response) {
+        var failureStore = (Map<String, Object>) response.get("failure_store");
+        return getIndices(failureStore);
+
+    }
+
+    @SuppressWarnings("unchecked")
+    private List<String> getIndices(Map<String, Object> response) {
+        List<Map<String, String>> indices = (List<Map<String, String>>) response.get("indices");
+        return indices.stream().map(index -> index.get("index_name")).toList();
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java
index 9839f9abb080e..619bfd74d853c 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java
@@ -28,6 +28,7 @@ public abstract class DisabledSecurityDataStreamTestCase extends ESRestTestCase
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
         .feature(FeatureFlag.FAILURE_STORE_ENABLED)
+        .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "false")
         .setting("xpack.watcher.enabled", "false")
         .build();
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java
deleted file mode 100644
index 8b29b1609711f..0000000000000
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.datastreams.logsdb.qa;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.logsdb.datageneration.DataGenerator;
-import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
-import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
-import org.elasticsearch.logsdb.datageneration.fields.PredefinedField;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.function.Consumer;
-
-public class DataGenerationHelper {
-    private final boolean keepArraySource;
-
-    private final DataGenerator dataGenerator;
-
-    public DataGenerationHelper() {
-        this(b -> {});
-    }
-
-    public DataGenerationHelper(Consumer<DataGeneratorSpecification.Builder> builderConfigurator) {
-        this.keepArraySource = ESTestCase.randomBoolean();
-
-        var specificationBuilder = DataGeneratorSpecification.builder()
-            .withFullyDynamicMapping(ESTestCase.randomBoolean())
-            .withPredefinedFields(
-                List.of(
-                    // Customized because it always needs doc_values for aggregations.
-                    new PredefinedField.WithGenerator("host.name", new FieldDataGenerator() {
-                        @Override
-                        public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-                            return b -> b.startObject().field("type", "keyword").endObject();
-                        }
-
-                        @Override
-                        public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-                            return b -> b.value(ESTestCase.randomAlphaOfLength(5));
-                        }
-                    }),
-                    // Needed for terms query
-                    new PredefinedField.WithGenerator("method", new FieldDataGenerator() {
-                        @Override
-                        public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-                            return b -> b.startObject().field("type", "keyword").endObject();
-                        }
-
-                        @Override
-                        public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-                            return b -> b.value(ESTestCase.randomFrom("put", "post", "get"));
-                        }
-                    }),
-
-                    // Needed for histogram aggregation
-                    new PredefinedField.WithGenerator("memory_usage_bytes", new FieldDataGenerator() {
-                        @Override
-                        public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-                            return b -> b.startObject().field("type", "long").endObject();
-                        }
-
-                        @Override
-                        public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-                            // We can generate this using standard long field but we would get "too many buckets"
-                            return b -> b.value(ESTestCase.randomLongBetween(1000, 2000));
-                        }
-                    })
-                )
-            );
-
-        // Customize builder if necessary
-        builderConfigurator.accept(specificationBuilder);
-
-        this.dataGenerator = new DataGenerator(specificationBuilder.build());
-    }
-
-    DataGenerator getDataGenerator() {
-        return dataGenerator;
-    }
-
-    void logsDbMapping(XContentBuilder builder) throws IOException {
-        dataGenerator.writeMapping(builder);
-    }
-
-    void standardMapping(XContentBuilder builder) throws IOException {
-        dataGenerator.writeMapping(builder);
-    }
-
-    void logsDbSettings(Settings.Builder builder) {
-        if (keepArraySource) {
-            builder.put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), "arrays");
-        }
-    }
-}
diff --git a/modules/data-streams/src/main/java/module-info.java b/modules/data-streams/src/main/java/module-info.java
index 16229f9eb2394..2d49029c1023c 100644
--- a/modules/data-streams/src/main/java/module-info.java
+++ b/modules/data-streams/src/main/java/module-info.java
@@ -17,6 +17,7 @@
     exports org.elasticsearch.datastreams.action to org.elasticsearch.server;
     exports org.elasticsearch.datastreams.lifecycle.action to org.elasticsearch.server;
     exports org.elasticsearch.datastreams.lifecycle;
+    exports org.elasticsearch.datastreams.options.action to org.elasticsearch.server;
 
     provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.datastreams.DataStreamFeatures;
 }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
index ab7e590b1631e..f60a3e5c47a7f 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
@@ -26,6 +26,7 @@
 public class DataStreamFeatures implements FeatureSpecification {
 
     public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
+    public static final NodeFeature DATA_STREAM_FAILURE_STORE_TSDB_FIX = new NodeFeature("data_stream.failure_store.tsdb_fix");
 
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
@@ -41,4 +42,9 @@ public Set<NodeFeature> getFeatures() {
             DataStreamGlobalRetention.GLOBAL_RETENTION                       // Added in 8.14
         );
     }
+
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX);
+    }
 }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java
index a3d0347c3d192..d6a0fd86265e5 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java
@@ -59,7 +59,7 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider {
     public Settings getAdditionalIndexSettings(
         String indexName,
         @Nullable String dataStreamName,
-        boolean isTimeSeries,
+        @Nullable IndexMode templateIndexMode,
         Metadata metadata,
         Instant resolvedAt,
         Settings indexTemplateAndCreateRequestSettings,
@@ -70,15 +70,16 @@ public Settings getAdditionalIndexSettings(
             // First backing index is created and then data stream is rolled over (in a single cluster state update).
             // So at this point we can't check index_mode==time_series,
             // so checking that index_mode==null|standard and templateIndexMode == TIME_SERIES
+            boolean isMigratingToTimeSeries = templateIndexMode == IndexMode.TIME_SERIES;
             boolean migrating = dataStream != null
                 && (dataStream.getIndexMode() == null || dataStream.getIndexMode() == IndexMode.STANDARD)
-                && isTimeSeries;
+                && isMigratingToTimeSeries;
             IndexMode indexMode;
             if (migrating) {
                 indexMode = IndexMode.TIME_SERIES;
             } else if (dataStream != null) {
-                indexMode = isTimeSeries ? dataStream.getIndexMode() : null;
-            } else if (isTimeSeries) {
+                indexMode = isMigratingToTimeSeries ? dataStream.getIndexMode() : null;
+            } else if (isMigratingToTimeSeries) {
                 indexMode = IndexMode.TIME_SERIES;
             } else {
                 indexMode = null;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
index 1a6465a251021..cb7445705537a 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction;
 import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction;
 import org.elasticsearch.client.internal.OriginSettingClient;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -56,6 +57,15 @@
 import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction;
 import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction;
 import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamLifecycleAction;
+import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.action.TransportDeleteDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.action.TransportGetDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.action.TransportPutDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.rest.RestDeleteDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.rest.RestGetDataStreamOptionsAction;
+import org.elasticsearch.datastreams.options.rest.RestPutDataStreamOptionsAction;
 import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction;
 import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction;
 import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction;
@@ -229,6 +239,11 @@ public Collection<?> createComponents(PluginServices services) {
         actions.add(new ActionHandler<>(DeleteDataStreamLifecycleAction.INSTANCE, TransportDeleteDataStreamLifecycleAction.class));
         actions.add(new ActionHandler<>(ExplainDataStreamLifecycleAction.INSTANCE, TransportExplainDataStreamLifecycleAction.class));
         actions.add(new ActionHandler<>(GetDataStreamLifecycleStatsAction.INSTANCE, TransportGetDataStreamLifecycleStatsAction.class));
+        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
+            actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class));
+            actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class));
+            actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class));
+        }
         return actions;
     }
 
@@ -261,6 +276,11 @@ public List<RestHandler> getRestHandlers(
         handlers.add(new RestDeleteDataStreamLifecycleAction());
         handlers.add(new RestExplainDataStreamLifecycleAction());
         handlers.add(new RestDataStreamLifecycleStatsAction());
+        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
+            handlers.add(new RestGetDataStreamOptionsAction());
+            handlers.add(new RestPutDataStreamOptionsAction());
+            handlers.add(new RestDeleteDataStreamOptionsAction());
+        }
         return handlers;
     }
 
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java
index b9f5bdea9c90e..edc17433ab746 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java
@@ -8,6 +8,8 @@
  */
 package org.elasticsearch.datastreams.action;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.datastreams.PromoteDataStreamAction;
@@ -23,6 +25,8 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.logging.HeaderWarning;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.indices.SystemIndices;
@@ -31,8 +35,12 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import static org.elasticsearch.core.Strings.format;
+
 public class PromoteDataStreamTransportAction extends AcknowledgedTransportMasterNodeAction<PromoteDataStreamAction.Request> {
 
+    private static final Logger logger = LogManager.getLogger(PromoteDataStreamTransportAction.class);
+
     private final SystemIndices systemIndices;
 
     @Inject
@@ -94,16 +102,41 @@ private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String
 
     static ClusterState promoteDataStream(ClusterState currentState, PromoteDataStreamAction.Request request) {
         DataStream dataStream = currentState.getMetadata().dataStreams().get(request.getName());
+
         if (dataStream == null) {
             throw new ResourceNotFoundException("data stream [" + request.getName() + "] does not exist");
         }
 
+        warnIfTemplateMissingForDatastream(dataStream, currentState);
+
         DataStream promotedDataStream = dataStream.promoteDataStream();
         Metadata.Builder metadata = Metadata.builder(currentState.metadata());
         metadata.put(promotedDataStream);
         return ClusterState.builder(currentState).metadata(metadata).build();
     }
 
+    private static void warnIfTemplateMissingForDatastream(DataStream dataStream, ClusterState currentState) {
+        var datastreamName = dataStream.getName();
+
+        var matchingIndex = currentState.metadata()
+            .templatesV2()
+            .values()
+            .stream()
+            .filter(cit -> cit.getDataStreamTemplate() != null)
+            .flatMap(cit -> cit.indexPatterns().stream())
+            .anyMatch(pattern -> Regex.simpleMatch(pattern, datastreamName));
+
+        if (matchingIndex == false) {
+            String warningMessage = format(
+                "Data stream [%s] does not have a matching index template. This will cause rollover to fail until a matching index "
+                    + "template is created",
+                datastreamName
+            );
+            logger.warn(() -> warningMessage);
+            HeaderWarning.addWarning(warningMessage);
+        }
+    }
+
     @Override
     protected ClusterBlockException checkBlock(PromoteDataStreamAction.Request request, ClusterState state) {
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
index 878583de4861f..7d2828e30d5ab 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
@@ -946,7 +946,7 @@ private Set<Index> maybeExecuteForceMerge(ClusterState state, List<Index> indice
                 UpdateSettingsRequest updateMergePolicySettingsRequest = new UpdateSettingsRequest();
                 updateMergePolicySettingsRequest.indicesOptions(
                     IndicesOptions.builder(updateMergePolicySettingsRequest.indicesOptions())
-                        .failureStoreOptions(new IndicesOptions.FailureStoreOptions(true, true))
+                        .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                         .build()
                 );
                 updateMergePolicySettingsRequest.indices(indexName);
@@ -1408,9 +1408,7 @@ static RolloverRequest getDefaultRolloverRequest(
         RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null).masterNodeTimeout(TimeValue.MAX_VALUE);
         if (rolloverFailureStore) {
             rolloverRequest.setIndicesOptions(
-                IndicesOptions.builder(rolloverRequest.indicesOptions())
-                    .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
-                    .build()
+                IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
             );
         }
         rolloverRequest.setConditions(rolloverConfiguration.resolveRolloverConditions(dataRetention));
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..98a29dd636ddf
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.datastreams.options.action;
+
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * Removes the data stream options configuration from the requested data streams.
+ */
+public class DeleteDataStreamOptionsAction {
+
+    public static final ActionType<AcknowledgedResponse> INSTANCE = new ActionType<>("indices:admin/data_stream/options/delete");
+
+    private DeleteDataStreamOptionsAction() {/* no instances */}
+
+    public static final class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable {
+
+        private String[] names;
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false)
+            )
+            .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true))
+            .build();
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.names = in.readOptionalStringArray();
+            this.indicesOptions = IndicesOptions.readIndicesOptions(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeOptionalStringArray(names);
+            indicesOptions.writeIndicesOptions(out);
+        }
+
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names) {
+            super(masterNodeTimeout, ackTimeout);
+            this.names = names;
+        }
+
+        public String[] getNames() {
+            return names;
+        }
+
+        @Override
+        public String[] indices() {
+            return names;
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return indicesOptions;
+        }
+
+        public Request indicesOptions(IndicesOptions indicesOptions) {
+            this.indicesOptions = indicesOptions;
+            return this;
+        }
+
+        @Override
+        public boolean includeDataStreams() {
+            return true;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Arrays.equals(names, request.names) && Objects.equals(indicesOptions, request.indicesOptions);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(indicesOptions);
+            result = 31 * result + Arrays.hashCode(names);
+            return result;
+        }
+
+        @Override
+        public IndicesRequest indices(String... indices) {
+            this.names = indices;
+            return this;
+        }
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..c1354da1129ca
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.options.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * This action retrieves the data stream options from every data stream. Currently, data stream options only support
+ * failure store.
+ */
+public class GetDataStreamOptionsAction {
+
+    public static final ActionType<Response> INSTANCE = new ActionType<>("indices:admin/data_stream/options/get");
+
+    private GetDataStreamOptionsAction() {/* no instances */}
+
+    public static class Request extends MasterNodeReadRequest<Request> implements IndicesRequest.Replaceable {
+
+        private String[] names;
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false)
+            )
+            .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true))
+            .build();
+        private boolean includeDefaults = false;
+
+        public Request(TimeValue masterNodeTimeout, String[] names) {
+            super(masterNodeTimeout);
+            this.names = names;
+        }
+
+        public Request(TimeValue masterNodeTimeout, String[] names, boolean includeDefaults) {
+            super(masterNodeTimeout);
+            this.names = names;
+            this.includeDefaults = includeDefaults;
+        }
+
+        public String[] getNames() {
+            return names;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.names = in.readOptionalStringArray();
+            this.indicesOptions = IndicesOptions.readIndicesOptions(in);
+            this.includeDefaults = in.readBoolean();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeOptionalStringArray(names);
+            indicesOptions.writeIndicesOptions(out);
+            out.writeBoolean(includeDefaults);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Arrays.equals(names, request.names)
+                && indicesOptions.equals(request.indicesOptions)
+                && includeDefaults == request.includeDefaults;
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(indicesOptions, includeDefaults);
+            result = 31 * result + Arrays.hashCode(names);
+            return result;
+        }
+
+        @Override
+        public String[] indices() {
+            return names;
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return indicesOptions;
+        }
+
+        public boolean includeDefaults() {
+            return includeDefaults;
+        }
+
+        public Request indicesOptions(IndicesOptions indicesOptions) {
+            this.indicesOptions = indicesOptions;
+            return this;
+        }
+
+        @Override
+        public boolean includeDataStreams() {
+            return true;
+        }
+
+        @Override
+        public IndicesRequest indices(String... indices) {
+            this.names = indices;
+            return this;
+        }
+
+        public Request includeDefaults(boolean includeDefaults) {
+            this.includeDefaults = includeDefaults;
+            return this;
+        }
+    }
+
+    public static class Response extends ActionResponse implements ChunkedToXContentObject {
+        public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams");
+
+        public record DataStreamEntry(String dataStreamName, DataStreamOptions dataStreamOptions) implements Writeable, ToXContentObject {
+
+            public static final ParseField NAME_FIELD = new ParseField("name");
+            public static final ParseField OPTIONS_FIELD = new ParseField("options");
+
+            DataStreamEntry(StreamInput in) throws IOException {
+                this(in.readString(), in.readOptionalWriteable(DataStreamOptions::read));
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                out.writeString(dataStreamName);
+                out.writeOptionalWriteable(dataStreamOptions);
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.startObject();
+                builder.field(NAME_FIELD.getPreferredName(), dataStreamName);
+                if (dataStreamOptions != null && dataStreamOptions.isEmpty() == false) {
+                    builder.field(OPTIONS_FIELD.getPreferredName(), dataStreamOptions);
+                }
+                builder.endObject();
+                return builder;
+            }
+        }
+
+        private final List<DataStreamEntry> dataStreams;
+
+        public Response(List<DataStreamEntry> dataStreams) {
+            this.dataStreams = dataStreams;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            this(in.readCollectionAsList(DataStreamEntry::new));
+        }
+
+        public List<DataStreamEntry> getDataStreams() {
+            return dataStreams;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeCollection(dataStreams);
+        }
+
+        @Override
+        public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+            return Iterators.concat(Iterators.single((builder, params) -> {
+                builder.startObject();
+                builder.startArray(DATA_STREAMS_FIELD.getPreferredName());
+                return builder;
+            }),
+                Iterators.map(dataStreams.iterator(), entry -> (builder, params) -> entry.toXContent(builder, outerParams)),
+                Iterators.single((builder, params) -> {
+                    builder.endArray();
+                    builder.endObject();
+                    return builder;
+                })
+            );
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response response = (Response) o;
+            return dataStreams.equals(response.dataStreams);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(dataStreams);
+        }
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..d055a6972312a
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.datastreams.options.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Sets the data stream options that was provided in the request to the requested data streams.
+ */
+public class PutDataStreamOptionsAction {
+
+    public static final ActionType<AcknowledgedResponse> INSTANCE = new ActionType<>("indices:admin/data_stream/options/put");
+
+    private PutDataStreamOptionsAction() {/* no instances */}
+
+    public static final class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable {
+
+        public interface Factory {
+            Request create(@Nullable DataStreamFailureStore dataStreamFailureStore);
+        }
+
+        public static final ConstructingObjectParser<Request, Factory> PARSER = new ConstructingObjectParser<>(
+            "put_data_stream_options_request",
+            false,
+            (args, factory) -> factory.create((DataStreamFailureStore) args[0])
+        );
+
+        static {
+            PARSER.declareObjectOrNull(
+                ConstructingObjectParser.optionalConstructorArg(),
+                (p, c) -> DataStreamFailureStore.PARSER.parse(p, null),
+                null,
+                new ParseField("failure_store")
+            );
+        }
+
+        public static Request parseRequest(XContentParser parser, Factory factory) {
+            return PARSER.apply(parser, factory);
+        }
+
+        private String[] names;
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false)
+            )
+            .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true))
+            .build();
+        private final DataStreamOptions options;
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.names = in.readStringArray();
+            this.indicesOptions = IndicesOptions.readIndicesOptions(in);
+            options = DataStreamOptions.read(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(names);
+            indicesOptions.writeIndicesOptions(out);
+            out.writeWriteable(options);
+        }
+
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamOptions options) {
+            super(masterNodeTimeout, ackTimeout);
+            this.names = names;
+            this.options = options;
+        }
+
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable DataStreamFailureStore failureStore) {
+            super(masterNodeTimeout, ackTimeout);
+            this.names = names;
+            this.options = new DataStreamOptions(failureStore);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+            if (options.failureStore() == null) {
+                validationException = addValidationError("At least one option needs to be provided", validationException);
+            }
+            return validationException;
+        }
+
+        public String[] getNames() {
+            return names;
+        }
+
+        public DataStreamOptions getOptions() {
+            return options;
+        }
+
+        @Override
+        public String[] indices() {
+            return names;
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return indicesOptions;
+        }
+
+        public Request indicesOptions(IndicesOptions indicesOptions) {
+            this.indicesOptions = indicesOptions;
+            return this;
+        }
+
+        @Override
+        public boolean includeDataStreams() {
+            return true;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Arrays.equals(names, request.names)
+                && Objects.equals(indicesOptions, request.indicesOptions)
+                && options.equals(request.options);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(indicesOptions, options);
+            result = 31 * result + Arrays.hashCode(names);
+            return result;
+        }
+
+        @Override
+        public IndicesRequest indices(String... names) {
+            this.names = names;
+            return this;
+        }
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..ead23ed78222b
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.options.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.List;
+
+/**
+ * Transport action that resolves the data stream names from the request and removes any configured data stream options from them.
+ */
+public class TransportDeleteDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction<DeleteDataStreamOptionsAction.Request> {
+
+    private final MetadataDataStreamsService metadataDataStreamsService;
+    private final SystemIndices systemIndices;
+
+    @Inject
+    public TransportDeleteDataStreamOptionsAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        MetadataDataStreamsService metadataDataStreamsService,
+        SystemIndices systemIndices
+    ) {
+        super(
+            DeleteDataStreamOptionsAction.INSTANCE.name(),
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            DeleteDataStreamOptionsAction.Request::new,
+            indexNameExpressionResolver,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.metadataDataStreamsService = metadataDataStreamsService;
+        this.systemIndices = systemIndices;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        DeleteDataStreamOptionsAction.Request request,
+        ClusterState state,
+        ActionListener<AcknowledgedResponse> listener
+    ) {
+        List<String> dataStreamNames = DataStreamsActionUtil.getDataStreamNames(
+            indexNameExpressionResolver,
+            state,
+            request.getNames(),
+            request.indicesOptions()
+        );
+        for (String name : dataStreamNames) {
+            systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext());
+        }
+        metadataDataStreamsService.removeDataStreamOptions(dataStreamNames, request.ackTimeout(), request.masterNodeTimeout(), listener);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(DeleteDataStreamOptionsAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..b032b35c943c0
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.options.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Collects the data streams from the cluster state and then returns for each data stream its name and its
+ * data stream options. Currently, data stream options include only the failure store configuration.
+ */
+public class TransportGetDataStreamOptionsAction extends TransportMasterNodeReadAction<
+    GetDataStreamOptionsAction.Request,
+    GetDataStreamOptionsAction.Response> {
+
+    private final SystemIndices systemIndices;
+
+    @Inject
+    public TransportGetDataStreamOptionsAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        SystemIndices systemIndices
+    ) {
+        super(
+            GetDataStreamOptionsAction.INSTANCE.name(),
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            GetDataStreamOptionsAction.Request::new,
+            indexNameExpressionResolver,
+            GetDataStreamOptionsAction.Response::new,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.systemIndices = systemIndices;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        GetDataStreamOptionsAction.Request request,
+        ClusterState state,
+        ActionListener<GetDataStreamOptionsAction.Response> listener
+    ) {
+        List<String> requestedDataStreams = DataStreamsActionUtil.getDataStreamNames(
+            indexNameExpressionResolver,
+            state,
+            request.getNames(),
+            request.indicesOptions()
+        );
+        Map<String, DataStream> dataStreams = state.metadata().dataStreams();
+        for (String name : requestedDataStreams) {
+            systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext());
+        }
+        listener.onResponse(
+            new GetDataStreamOptionsAction.Response(
+                requestedDataStreams.stream()
+                    .map(dataStreams::get)
+                    .filter(Objects::nonNull)
+                    .map(
+                        dataStream -> new GetDataStreamOptionsAction.Response.DataStreamEntry(
+                            dataStream.getName(),
+                            dataStream.getDataStreamOptions()
+                        )
+                    )
+                    .sorted(Comparator.comparing(GetDataStreamOptionsAction.Response.DataStreamEntry::dataStreamName))
+                    .toList()
+            )
+        );
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(GetDataStreamOptionsAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..b1386232c44f9
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.options.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.List;
+
+/**
+ * Transport action that resolves the data stream names from the request and sets the data stream lifecycle provided in the request.
+ */
+public class TransportPutDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction<PutDataStreamOptionsAction.Request> {
+
+    private final MetadataDataStreamsService metadataDataStreamsService;
+    private final SystemIndices systemIndices;
+
+    @Inject
+    public TransportPutDataStreamOptionsAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        MetadataDataStreamsService metadataDataStreamsService,
+        SystemIndices systemIndices
+    ) {
+        super(
+            PutDataStreamOptionsAction.INSTANCE.name(),
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            PutDataStreamOptionsAction.Request::new,
+            indexNameExpressionResolver,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.metadataDataStreamsService = metadataDataStreamsService;
+        this.systemIndices = systemIndices;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        PutDataStreamOptionsAction.Request request,
+        ClusterState state,
+        ActionListener<AcknowledgedResponse> listener
+    ) {
+        List<String> dataStreamNames = DataStreamsActionUtil.getDataStreamNames(
+            indexNameExpressionResolver,
+            state,
+            request.getNames(),
+            request.indicesOptions()
+        );
+        for (String name : dataStreamNames) {
+            systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext());
+        }
+        metadataDataStreamsService.setDataStreamOptions(
+            dataStreamNames,
+            request.getOptions(),
+            request.ackTimeout(),
+            request.masterNodeTimeout(),
+            listener
+        );
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(PutDataStreamOptionsAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..96460632ff443
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.options.rest;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+
+@ServerlessScope(Scope.INTERNAL)
+public class RestDeleteDataStreamOptionsAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "delete_data_stream_options_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(DELETE, "/_data_stream/{name}/_options"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        final var deleteDataOptionsRequest = new DeleteDataStreamOptionsAction.Request(
+            getMasterNodeTimeout(request),
+            request.paramAsTime("timeout", AcknowledgedRequest.DEFAULT_ACK_TIMEOUT),
+            Strings.splitStringByCommaToArray(request.param("name"))
+        );
+        deleteDataOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteDataOptionsRequest.indicesOptions()));
+        return channel -> client.execute(
+            DeleteDataStreamOptionsAction.INSTANCE,
+            deleteDataOptionsRequest,
+            new RestToXContentListener<>(channel)
+        );
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..6d6530efce1b9
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.options.rest;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
+
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestGetDataStreamOptionsAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "get_data_stream_options_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(GET, "/_data_stream/{name}/_options"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        GetDataStreamOptionsAction.Request getDataStreamOptionsRequest = new GetDataStreamOptionsAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
+            Strings.splitStringByCommaToArray(request.param("name"))
+        );
+        getDataStreamOptionsRequest.includeDefaults(request.paramAsBoolean("include_defaults", false));
+        getDataStreamOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, getDataStreamOptionsRequest.indicesOptions()));
+        return channel -> client.execute(
+            GetDataStreamOptionsAction.INSTANCE,
+            getDataStreamOptionsRequest,
+            new RestRefCountedChunkedToXContentListener<>(channel)
+        );
+    }
+
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java
new file mode 100644
index 0000000000000..9191b96b6039e
--- /dev/null
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.options.rest;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+import static org.elasticsearch.rest.RestUtils.getAckTimeout;
+import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestPutDataStreamOptionsAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "put_data_stream_options_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(PUT, "/_data_stream/{name}/_options"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        try (XContentParser parser = request.contentParser()) {
+            PutDataStreamOptionsAction.Request putOptionsRequest = PutDataStreamOptionsAction.Request.parseRequest(
+                parser,
+                (failureStore) -> new PutDataStreamOptionsAction.Request(
+                    getMasterNodeTimeout(request),
+                    getAckTimeout(request),
+                    Strings.splitStringByCommaToArray(request.param("name")),
+                    failureStore
+                )
+            );
+            putOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, putOptionsRequest.indicesOptions()));
+            return channel -> client.execute(PutDataStreamOptionsAction.INSTANCE, putOptionsRequest, new RestToXContentListener<>(channel));
+        }
+    }
+}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
index 3456f4b679474..b61e38297397d 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
@@ -43,7 +43,7 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
                 IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED,
                 "verbose"
             ),
-            DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FailureStoreOptions.FAILURE_STORE) : Set.of()
+            DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FAILURE_STORE_QUERY_PARAM) : Set.of()
         )
     );
 
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
index d8d4a9c03933a..015752724cb5d 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
@@ -78,7 +78,7 @@ public void testGetAdditionalIndexSettings() throws Exception {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -123,7 +123,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -193,7 +193,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -218,7 +218,7 @@ public void testGetAdditionalIndexSettingsNoMappings() {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -243,7 +243,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -268,7 +268,7 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -299,7 +299,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce
         var result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -336,7 +336,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi
             () -> provider.getAdditionalIndexSettings(
                 DataStream.getDefaultBackingIndexName(dataStreamName, 1),
                 dataStreamName,
-                true,
+                IndexMode.TIME_SERIES,
                 metadata,
                 now,
                 settings,
@@ -362,7 +362,7 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            false,
+            null,
             metadata,
             Instant.ofEpochMilli(1L),
             settings,
@@ -382,7 +382,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 2),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
@@ -415,7 +415,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 2),
             dataStreamName,
-            false,
+            null,
             metadata,
             Instant.ofEpochMilli(1L),
             settings,
@@ -694,7 +694,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce
         var result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            true,
+            IndexMode.TIME_SERIES,
             metadata,
             now,
             settings,
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
index 0d5ce54c44b56..698ab427ab040 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
@@ -225,17 +225,11 @@ public void testOperationsExecutedOnce() {
         assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
         assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(true, false))
-        );
+        assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA));
         assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
         assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(false, true))
-        );
+        assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES));
         List<DeleteIndexRequest> deleteRequests = clientSeenRequests.subList(2, 5)
             .stream()
             .map(transportRequest -> (DeleteIndexRequest) transportRequest)
@@ -1552,17 +1546,11 @@ public void testFailureStoreIsManagedEvenWhenDisabled() {
         assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
         assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(true, false))
-        );
+        assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA));
         assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
         assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(false, true))
-        );
+        assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES));
         assertThat(
             ((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0],
             is(dataStream.getFailureIndices().getIndices().get(0).getName())
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
index 56f387c016261..de5cf3baa744e 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
@@ -182,6 +182,107 @@ index without timestamp:
         body:
           - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
 
+---
+TSDB failures go to failure store:
+  - requires:
+      cluster_features: ["data_stream.failure_store.tsdb_fix"]
+      reason: "tests tsdb failure store fixes in 8.16.0 that catch timestamp errors that happen earlier in the process and redirect them to the failure store."
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template2] has index patterns [fs-k8s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [ "fs-k8s*" ]
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              index:
+                mode: time_series
+                number_of_replicas: 1
+                number_of_shards: 2
+                routing_path: [ metricset, time_series_dimension ]
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
+            mappings:
+              properties:
+                "@timestamp":
+                  type: date
+                metricset:
+                  type: keyword
+                  time_series_dimension: true
+                k8s:
+                  properties:
+                    pod:
+                      properties:
+                        uid:
+                          type: keyword
+                          time_series_dimension: true
+                        name:
+                          type: keyword
+                        ip:
+                          type: ip
+                        network:
+                          properties:
+                            tx:
+                              type: long
+                            rx:
+                              type: long
+  - do:
+      index:
+        index: fs-k8s
+        body:
+          - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+  - match: { result : "created"}
+  - match: { failure_store : "used"}
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "create": { "_index": "fs-k8s"} }'
+          - '{"@timestamp":"2021-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "k8s"} }'
+          - '{ "@timestamp": "2021-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "fs-k8s"} }'
+          - '{ "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "fs-k8s"} }'
+          - '{ "@timestamp":"2000-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "k8s"} }'
+          - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "k8s"} }'
+          - '{ "@timestamp":"2000-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+  - is_true: errors
+
+  # Successfully indexed to backing index
+  - match: { items.0.create._index: '/\.ds-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { items.0.create.status: 201 }
+  - is_false: items.0.create.failure_store
+  - match: { items.1.create._index: '/\.ds-k8s-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { items.1.create.status: 201 }
+  - is_false: items.1.create.failure_store
+
+  # Successfully indexed to failure store
+  - match: { items.2.create._index: '/\.fs-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - match: { items.2.create.status: 201 }
+  - match: { items.2.create.failure_store: used }
+  - match: { items.3.create._index: '/\.fs-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - match: { items.3.create.status: 201 }
+  - match: { items.3.create.failure_store: used }
+
+  # Rejected, eligible to go to failure store, but failure store not enabled
+  - match: { items.4.create._index: 'k8s' }
+  - match: { items.4.create.status: 400 }
+  - match: { items.4.create.error.type: timestamp_error }
+  - match: { items.4.create.failure_store: not_enabled }
+  - match: { items.4.create._index: 'k8s' }
+  - match: { items.4.create.status: 400 }
+  - match: { items.4.create.error.type: timestamp_error }
+  - match: { items.4.create.failure_store: not_enabled }
+
 ---
 index without timestamp with pipeline:
   - do:
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml
index cb5578a282dc9..9b5a9dae8bc0a 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml
@@ -879,7 +879,7 @@ teardown:
   # Successfully indexed to backing index
   - match: { items.0.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
   - match: { items.0.create.status: 201 }
-  - is_false: items.1.create.failure_store
+  - is_false: items.0.create.failure_store
 
   # Rejected but not eligible to go to failure store
   - match: { items.1.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java
index e2c75a6401187..fc8d701b953f6 100644
--- a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java
+++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java
@@ -56,6 +56,7 @@ public abstract class DotPrefixValidator<RequestType> implements MappedActionFil
      *
      * .elastic-connectors-* is used by enterprise search
      * .ml-* is used by ML
+     * .slo-observability-* is used by Observability
      */
     private static Set<String> IGNORED_INDEX_NAMES = Set.of(
         ".elastic-connectors-v1",
@@ -63,7 +64,11 @@ public abstract class DotPrefixValidator<RequestType> implements MappedActionFil
         ".ml-state",
         ".ml-anomalies-unrelated"
     );
-    private static Set<Pattern> IGNORED_INDEX_PATTERNS = Set.of(Pattern.compile("\\.ml-state-\\d+"));
+    private static Set<Pattern> IGNORED_INDEX_PATTERNS = Set.of(
+        Pattern.compile("\\.ml-state-\\d+"),
+        Pattern.compile("\\.slo-observability\\.sli-v\\d+.*"),
+        Pattern.compile("\\.slo-observability\\.summary-v\\d+.*")
+    );
 
     DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class);
 
@@ -99,10 +104,11 @@ void validateIndices(@Nullable Set<String> indices) {
                 if (Strings.hasLength(index)) {
                     char c = getFirstChar(index);
                     if (c == '.') {
-                        if (IGNORED_INDEX_NAMES.contains(index)) {
+                        final String strippedName = stripDateMath(index);
+                        if (IGNORED_INDEX_NAMES.contains(strippedName)) {
                             return;
                         }
-                        if (IGNORED_INDEX_PATTERNS.stream().anyMatch(p -> p.matcher(index).matches())) {
+                        if (IGNORED_INDEX_PATTERNS.stream().anyMatch(p -> p.matcher(strippedName).matches())) {
                             return;
                         }
                         deprecationLogger.warn(
@@ -132,7 +138,18 @@ private static char getFirstChar(String index) {
         return c;
     }
 
-    private boolean isInternalRequest() {
+    private static String stripDateMath(String index) {
+        char c = index.charAt(0);
+        if (c == '<') {
+            assert index.charAt(index.length() - 1) == '>'
+                : "expected index name with date math to start with < and end with >, how did this pass request validation? " + index;
+            return index.substring(1, index.length() - 1);
+        } else {
+            return index;
+        }
+    }
+
+    boolean isInternalRequest() {
         final String actionOrigin = threadContext.getTransient(ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME);
         final boolean isSystemContext = threadContext.isSystemContext();
         final boolean isInternalOrigin = Optional.ofNullable(actionOrigin).map(Strings::hasText).orElse(false);
diff --git a/modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java b/modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java
new file mode 100644
index 0000000000000..9adb33d51f510
--- /dev/null
+++ b/modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.BeforeClass;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DotPrefixValidatorTests extends ESTestCase {
+    private final OperatorValidator<?> opV = new OperatorValidator<>();
+    private final NonOperatorValidator<?> nonOpV = new NonOperatorValidator<>();
+    private static final Set<Setting<?>> settings;
+
+    private static ClusterService clusterService;
+    private static ClusterSettings clusterSettings;
+
+    static {
+        Set<Setting<?>> cSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        cSettings.add(DotPrefixValidator.VALIDATE_DOT_PREFIXES);
+        settings = cSettings;
+    }
+
+    @BeforeClass
+    public static void beforeClass() {
+        clusterService = mock(ClusterService.class);
+        clusterSettings = new ClusterSettings(Settings.EMPTY, Sets.newHashSet(DotPrefixValidator.VALIDATE_DOT_PREFIXES));
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
+        when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class));
+    }
+
+    public void testValidation() {
+
+        nonOpV.validateIndices(Set.of("regular"));
+        opV.validateIndices(Set.of("regular"));
+        assertFails(Set.of(".regular"));
+        opV.validateIndices(Set.of(".regular"));
+        assertFails(Set.of("first", ".second"));
+        assertFails(Set.of("<.regular-{MM-yy-dd}>"));
+
+        // Test ignored names
+        nonOpV.validateIndices(Set.of(".elastic-connectors-v1"));
+        nonOpV.validateIndices(Set.of(".elastic-connectors-sync-jobs-v1"));
+        nonOpV.validateIndices(Set.of(".ml-state"));
+        nonOpV.validateIndices(Set.of(".ml-anomalies-unrelated"));
+
+        // Test ignored patterns
+        nonOpV.validateIndices(Set.of(".ml-state-21309"));
+        nonOpV.validateIndices(Set.of(">.ml-state-21309>"));
+        nonOpV.validateIndices(Set.of(".slo-observability.sli-v2"));
+        nonOpV.validateIndices(Set.of(".slo-observability.sli-v2.3"));
+        nonOpV.validateIndices(Set.of(".slo-observability.sli-v2.3-2024-01-01"));
+        nonOpV.validateIndices(Set.of("<.slo-observability.sli-v3.3.{2024-10-16||/M{yyyy-MM-dd|UTC}}>"));
+        nonOpV.validateIndices(Set.of(".slo-observability.summary-v2"));
+        nonOpV.validateIndices(Set.of(".slo-observability.summary-v2.3"));
+        nonOpV.validateIndices(Set.of(".slo-observability.summary-v2.3-2024-01-01"));
+        nonOpV.validateIndices(Set.of("<.slo-observability.summary-v3.3.{2024-10-16||/M{yyyy-MM-dd|UTC}}>"));
+    }
+
+    private void assertFails(Set<String> indices) {
+        nonOpV.validateIndices(indices);
+        assertWarnings(
+            "Index ["
+                + indices.stream().filter(i -> i.startsWith(".") || i.startsWith("<.")).toList().getFirst()
+                + "] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version."
+        );
+    }
+
+    private class NonOperatorValidator<R> extends DotPrefixValidator<R> {
+
+        private NonOperatorValidator() {
+            super(new ThreadContext(Settings.EMPTY), clusterService);
+        }
+
+        @Override
+        protected Set<String> getIndicesFromRequest(Object request) {
+            return Set.of();
+        }
+
+        @Override
+        public String actionName() {
+            return "";
+        }
+
+        @Override
+        boolean isInternalRequest() {
+            return false;
+        }
+    }
+
+    private class OperatorValidator<R> extends NonOperatorValidator<R> {
+        @Override
+        boolean isInternalRequest() {
+            return true;
+        }
+    }
+}
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java
index 0bc7114f626c4..ad1fce0ca689a 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -40,18 +41,24 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER;
+import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING;
 import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING;
 import static org.hamcrest.Matchers.equalTo;
 
 public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase {
 
-    private static final String DATABASE_TYPE = "GeoIP2-City";
+    private static final String MAXMIND_DATABASE_TYPE = "GeoIP2-City";
+    private static final String IPINFO_DATABASE_TYPE = "asn";
 
     @ClassRule
-    public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(DATABASE_TYPE);
+    public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(
+        List.of(MAXMIND_DATABASE_TYPE),
+        List.of(IPINFO_DATABASE_TYPE)
+    );
 
     protected String getEndpoint() {
         return fixture.getAddress();
@@ -61,6 +68,7 @@ protected String getEndpoint() {
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString(MAXMIND_LICENSE_KEY_SETTING.getKey(), "license_key");
+        secureSettings.setString(IPINFO_TOKEN_SETTING.getKey(), "token");
         Settings.Builder builder = Settings.builder();
         builder.setSecureSettings(secureSettings)
             .put(super.nodeSettings(nodeOrdinal, otherSettings))
@@ -87,22 +95,27 @@ public void testEnterpriseDownloaderTask() throws Exception {
          * Note that the "enterprise database" is actually just a geolite database being loaded by the GeoIpHttpFixture.
          */
         EnterpriseGeoIpDownloader.DEFAULT_MAXMIND_ENDPOINT = getEndpoint();
-        final String pipelineName = "enterprise_geoip_pipeline";
+        EnterpriseGeoIpDownloader.DEFAULT_IPINFO_ENDPOINT = getEndpoint();
         final String indexName = "enterprise_geoip_test_index";
+        final String geoipPipelineName = "enterprise_geoip_pipeline";
+        final String iplocationPipelineName = "enterprise_iplocation_pipeline";
         final String sourceField = "ip";
-        final String targetField = "ip-city";
+        final String targetField = "ip-result";
 
         startEnterpriseGeoIpDownloaderTask();
-        configureDatabase(DATABASE_TYPE);
-        createGeoIpPipeline(pipelineName, DATABASE_TYPE, sourceField, targetField);
+        configureMaxmindDatabase(MAXMIND_DATABASE_TYPE);
+        configureIpinfoDatabase(IPINFO_DATABASE_TYPE);
+        waitAround();
+        createPipeline(geoipPipelineName, "geoip", MAXMIND_DATABASE_TYPE, sourceField, targetField);
+        createPipeline(iplocationPipelineName, "ip_location", IPINFO_DATABASE_TYPE, sourceField, targetField);
 
+        /*
+         * We know that the databases index has been populated (because we waited around, :wink:), but we don't know for sure that
+         * the databases have been pulled down and made available on all nodes. So we run these ingest-and-check steps in assertBusy blocks.
+         */
         assertBusy(() -> {
-            /*
-             * We know that the .geoip_databases index has been populated, but we don't know for sure that the database has been pulled
-             * down and made available on all nodes. So we run this ingest-and-check step in an assertBusy.
-             */
             logger.info("Ingesting a test document");
-            String documentId = ingestDocument(indexName, pipelineName, sourceField);
+            String documentId = ingestDocument(indexName, geoipPipelineName, sourceField, "89.160.20.128");
             GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet();
             Map<String, Object> returnedSource = getResponse.getSource();
             assertNotNull(returnedSource);
@@ -110,6 +123,16 @@ public void testEnterpriseDownloaderTask() throws Exception {
             assertNotNull(targetFieldValue);
             assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("Bredband2 AB"));
         });
+        assertBusy(() -> {
+            logger.info("Ingesting another test document");
+            String documentId = ingestDocument(indexName, iplocationPipelineName, sourceField, "12.10.66.1");
+            GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet();
+            Map<String, Object> returnedSource = getResponse.getSource();
+            assertNotNull(returnedSource);
+            Object targetFieldValue = returnedSource.get(targetField);
+            assertNotNull(targetFieldValue);
+            assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("OAKLAWN JOCKEY CLUB, INC."));
+        });
     }
 
     private void startEnterpriseGeoIpDownloaderTask() {
@@ -128,36 +151,53 @@ private void startEnterpriseGeoIpDownloaderTask() {
         );
     }
 
-    private void configureDatabase(String databaseType) throws Exception {
+    private void configureMaxmindDatabase(String databaseType) {
         admin().cluster()
             .execute(
                 PutDatabaseConfigurationAction.INSTANCE,
                 new PutDatabaseConfigurationAction.Request(
                     TimeValue.MAX_VALUE,
                     TimeValue.MAX_VALUE,
-                    new DatabaseConfiguration("test", databaseType, new DatabaseConfiguration.Maxmind("test_account"))
+                    new DatabaseConfiguration("test-1", databaseType, new DatabaseConfiguration.Maxmind("test_account"))
                 )
             )
             .actionGet();
+    }
+
+    private void configureIpinfoDatabase(String databaseType) {
+        admin().cluster()
+            .execute(
+                PutDatabaseConfigurationAction.INSTANCE,
+                new PutDatabaseConfigurationAction.Request(
+                    TimeValue.MAX_VALUE,
+                    TimeValue.MAX_VALUE,
+                    new DatabaseConfiguration("test-2", databaseType, new DatabaseConfiguration.Ipinfo())
+                )
+            )
+            .actionGet();
+    }
+
+    private void waitAround() throws Exception {
         ensureGreen(GeoIpDownloader.DATABASES_INDEX);
         assertBusy(() -> {
             SearchResponse searchResponse = client().search(new SearchRequest(GeoIpDownloader.DATABASES_INDEX)).actionGet();
             try {
-                assertThat(searchResponse.getHits().getHits().length, equalTo(1));
+                assertThat(searchResponse.getHits().getHits().length, equalTo(2));
             } finally {
                 searchResponse.decRef();
             }
         });
     }
 
-    private void createGeoIpPipeline(String pipelineName, String databaseType, String sourceField, String targetField) throws IOException {
+    private void createPipeline(String pipelineName, String processorType, String databaseType, String sourceField, String targetField)
+        throws IOException {
         putJsonPipeline(pipelineName, (builder, params) -> {
             builder.field("description", "test");
             builder.startArray("processors");
             {
                 builder.startObject();
                 {
-                    builder.startObject("geoip");
+                    builder.startObject(processorType);
                     {
                         builder.field("field", sourceField);
                         builder.field("target_field", targetField);
@@ -171,11 +211,11 @@ private void createGeoIpPipeline(String pipelineName, String databaseType, Strin
         });
     }
 
-    private String ingestDocument(String indexName, String pipelineName, String sourceField) {
+    private String ingestDocument(String indexName, String pipelineName, String sourceField, String value) {
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(
-            new IndexRequest(indexName).source("{\"" + sourceField + "\": \"89.160.20.128\"}", XContentType.JSON).setPipeline(pipelineName)
-        );
+        bulkRequest.add(new IndexRequest(indexName).source(Strings.format("""
+            { "%s": "%s"}
+            """, sourceField, value), XContentType.JSON).setPipeline(pipelineName));
         BulkResponse response = client().bulk(bulkRequest).actionGet();
         BulkItemResponse[] bulkItemResponses = response.getItems();
         assertThat(bulkItemResponses.length, equalTo(1));
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
index 6942cc3733d1e..f8c8d2bd359f3 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
@@ -256,8 +256,8 @@ public void testGeoIpDatabasesDownload() throws Exception {
                         res -> {
                             try {
                                 TotalHits totalHits = res.getHits().getTotalHits();
-                                assertEquals(TotalHits.Relation.EQUAL_TO, totalHits.relation);
-                                assertEquals(size, totalHits.value);
+                                assertEquals(TotalHits.Relation.EQUAL_TO, totalHits.relation());
+                                assertEquals(size, totalHits.value());
                                 assertEquals(size, res.getHits().getHits().length);
 
                                 List<byte[]> data = new ArrayList<>();
diff --git a/modules/ingest-geoip/src/main/java/module-info.java b/modules/ingest-geoip/src/main/java/module-info.java
index f64c30c060b08..0703d9fb449aa 100644
--- a/modules/ingest-geoip/src/main/java/module-info.java
+++ b/modules/ingest-geoip/src/main/java/module-info.java
@@ -18,4 +18,6 @@
 
     exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server;
     exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server;
+
+    exports org.elasticsearch.ingest.geoip to com.maxmind.db;
 }
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java
index 865d0c5a9eca0..3d2b54b04695f 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java
@@ -73,14 +73,14 @@ void updateDatabase(Path file, boolean update) {
         String databaseFileName = file.getFileName().toString();
         try {
             if (update) {
-                logger.info("database file changed [{}], reload database...", file);
+                logger.info("database file changed [{}], reloading database...", file);
                 DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, null);
                 DatabaseReaderLazyLoader existing = configDatabases.put(databaseFileName, loader);
                 if (existing != null) {
                     existing.shutdown();
                 }
             } else {
-                logger.info("database file removed [{}], close database...", file);
+                logger.info("database file removed [{}], closing database...", file);
                 DatabaseReaderLazyLoader existing = configDatabases.remove(databaseFileName);
                 assert existing != null;
                 existing.shutdown();
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java
index fa46540e29f7a..e04014ff693be 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java
@@ -38,6 +38,8 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.Closeable;
@@ -56,6 +58,7 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_SETTINGS_PREFIX;
 import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_SETTINGS_PREFIX;
 
 /**
@@ -71,6 +74,9 @@ public class EnterpriseGeoIpDownloader extends AllocatedPersistentTask {
     // a sha256 checksum followed by two spaces followed by an (ignored) file name
     private static final Pattern SHA256_CHECKSUM_PATTERN = Pattern.compile("(\\w{64})\\s\\s(.*)");
 
+    // an md5 checksum
+    private static final Pattern MD5_CHECKSUM_PATTERN = Pattern.compile("(\\w{32})");
+
     // for overriding in tests
     static String DEFAULT_MAXMIND_ENDPOINT = System.getProperty(
         MAXMIND_SETTINGS_PREFIX + "endpoint.default", //
@@ -79,6 +85,14 @@ public class EnterpriseGeoIpDownloader extends AllocatedPersistentTask {
     // n.b. a future enhancement might be to allow for a MAXMIND_ENDPOINT_SETTING, but
     // at the moment this is an unsupported system property for use in tests (only)
 
+    // for overriding in tests
+    static String DEFAULT_IPINFO_ENDPOINT = System.getProperty(
+        IPINFO_SETTINGS_PREFIX + "endpoint.default", //
+        "https://ipinfo.io/data"
+    );
+    // n.b. a future enhancement might be to allow for an IPINFO_ENDPOINT_SETTING, but
+    // at the moment this is an unsupported system property for use in tests (only)
+
     static final String DATABASES_INDEX = ".geoip_databases";
     static final int MAX_CHUNK_SIZE = 1024 * 1024;
 
@@ -236,7 +250,7 @@ boolean processDatabase(String id, DatabaseConfiguration database) throws IOExce
         logger.debug("Processing database [{}] for configuration [{}]", name, database.id());
 
         try (ProviderDownload downloader = downloaderFor(database)) {
-            if (downloader.validCredentials()) {
+            if (downloader != null && downloader.validCredentials()) {
                 // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension,
                 // but the downloading and indexing of database code expects it to be there, so we add it on here before continuing
                 final String fileName = name + ".mmdb";
@@ -444,9 +458,15 @@ private void scheduleNextRun(TimeValue time) {
     }
 
     private ProviderDownload downloaderFor(DatabaseConfiguration database) {
-        assert database.provider() instanceof DatabaseConfiguration.Maxmind
-            : "Attempt to use maxmind downloader with a provider of type" + database.provider().getClass();
-        return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider());
+        if (database.provider() instanceof DatabaseConfiguration.Maxmind maxmind) {
+            return new MaxmindDownload(database.name(), maxmind);
+        } else if (database.provider() instanceof DatabaseConfiguration.Ipinfo ipinfo) {
+            return new IpinfoDownload(database.name(), ipinfo);
+        } else {
+            throw new IllegalArgumentException(
+                Strings.format("Unexpected provider [%s] for configuration [%s]", database.provider().getClass(), database.id())
+            );
+        }
     }
 
     class MaxmindDownload implements ProviderDownload {
@@ -480,7 +500,7 @@ public HttpClient.PasswordAuthenticationHolder buildCredentials() {
 
         @Override
         public boolean validCredentials() {
-            return auth.get() != null;
+            return auth != null && auth.get() != null;
         }
 
         @Override
@@ -521,7 +541,101 @@ public CheckedSupplier<InputStream, IOException> download() {
 
         @Override
         public void close() throws IOException {
-            auth.close();
+            if (auth != null) auth.close();
+        }
+    }
+
+    class IpinfoDownload implements ProviderDownload {
+
+        final String name;
+        final DatabaseConfiguration.Ipinfo ipinfo;
+        HttpClient.PasswordAuthenticationHolder auth;
+
+        IpinfoDownload(String name, DatabaseConfiguration.Ipinfo ipinfo) {
+            this.name = name;
+            this.ipinfo = ipinfo;
+            this.auth = buildCredentials();
+        }
+
+        @Override
+        public HttpClient.PasswordAuthenticationHolder buildCredentials() {
+            final char[] tokenChars = tokenProvider.apply("ipinfo");
+
+            // if the token is missing or empty, return null as 'no auth'
+            if (tokenChars == null || tokenChars.length == 0) {
+                return null;
+            }
+
+            // ipinfo uses the token as the username component of basic auth, see https://ipinfo.io/developers#authentication
+            return new HttpClient.PasswordAuthenticationHolder(new String(tokenChars), new char[] {});
+        }
+
+        @Override
+        public boolean validCredentials() {
+            return auth != null && auth.get() != null;
+        }
+
+        private static final Set<String> FREE_DATABASES = Set.of("asn", "country", "country_asn");
+
+        @Override
+        public String url(String suffix) {
+            // note: the 'free' databases are in the sub-path 'free/' in terms of the download endpoint
+            final String internalName;
+            if (FREE_DATABASES.contains(name)) {
+                internalName = "free/" + name;
+            } else {
+                internalName = name;
+            }
+
+            // reminder, we're passing the ipinfo token as the username part of http basic auth,
+            // see https://ipinfo.io/developers#authentication
+
+            String endpointPattern = DEFAULT_IPINFO_ENDPOINT;
+            if (endpointPattern.contains("%")) {
+                throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]");
+            }
+            if (endpointPattern.endsWith("/") == false) {
+                endpointPattern += "/";
+            }
+            endpointPattern += "%s.%s";
+
+            // at this point the pattern looks like this (in the default case):
+            // https://ipinfo.io/data/%s.%s
+            // also see https://ipinfo.io/developers/database-download,
+            // and https://ipinfo.io/developers/database-filename-reference for more
+
+            return Strings.format(endpointPattern, internalName, suffix);
+        }
+
+        @Override
+        public Checksum checksum() throws IOException {
+            final String checksumJsonUrl = this.url("mmdb/checksums"); // a minor abuse of the idea of a 'suffix', :shrug:
+            byte[] data = httpClient.getBytes(auth.get(), checksumJsonUrl); // this throws if the auth is bad
+            Map<String, Object> checksums;
+            try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, data)) {
+                checksums = parser.map();
+            }
+            @SuppressWarnings("unchecked")
+            String md5 = ((Map<String, String>) checksums.get("checksums")).get("md5");
+            logger.trace("checksum was [{}]", md5);
+
+            var matcher = MD5_CHECKSUM_PATTERN.matcher(md5);
+            boolean match = matcher.matches();
+            if (match == false) {
+                throw new RuntimeException("Unexpected md5 response from [" + checksumJsonUrl + "]");
+            }
+            return Checksum.md5(md5);
+        }
+
+        @Override
+        public CheckedSupplier<InputStream, IOException> download() {
+            final String mmdbUrl = this.url("mmdb");
+            return () -> httpClient.get(auth.get(), mmdbUrl);
+        }
+
+        @Override
+        public void close() throws IOException {
+            if (auth != null) auth.close();
         }
     }
 
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java
index 5214c0e4a6a51..ae9bb109a3bf8 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java
@@ -54,11 +54,15 @@ public class EnterpriseGeoIpDownloaderTaskExecutor extends PersistentTasksExecut
 
     static final String MAXMIND_SETTINGS_PREFIX = "ingest.geoip.downloader.maxmind.";
 
+    static final String IPINFO_SETTINGS_PREFIX = "ingest.ip_location.downloader.ipinfo.";
+
     public static final Setting<SecureString> MAXMIND_LICENSE_KEY_SETTING = SecureSetting.secureString(
         MAXMIND_SETTINGS_PREFIX + "license_key",
         null
     );
 
+    public static final Setting<SecureString> IPINFO_TOKEN_SETTING = SecureSetting.secureString(IPINFO_SETTINGS_PREFIX + "token", null);
+
     private final Client client;
     private final HttpClient httpClient;
     private final ClusterService clusterService;
@@ -106,6 +110,10 @@ private char[] getSecureToken(final String type) {
             if (cachedSecureSettings.getSettingNames().contains(MAXMIND_LICENSE_KEY_SETTING.getKey())) {
                 token = cachedSecureSettings.getString(MAXMIND_LICENSE_KEY_SETTING.getKey()).getChars();
             }
+        } else if (type.equals("ipinfo")) {
+            if (cachedSecureSettings.getSettingNames().contains(IPINFO_TOKEN_SETTING.getKey())) {
+                token = cachedSecureSettings.getString(IPINFO_TOKEN_SETTING.getKey()).getChars();
+            }
         }
         return token;
     }
@@ -166,7 +174,7 @@ public synchronized void reload(Settings settings) {
         // `SecureSettings` are available here! cache them as they will be needed
         // whenever dynamic cluster settings change and we have to rebuild the accounts
         try {
-            this.cachedSecureSettings = extractSecureSettings(settings, List.of(MAXMIND_LICENSE_KEY_SETTING));
+            this.cachedSecureSettings = extractSecureSettings(settings, List.of(MAXMIND_LICENSE_KEY_SETTING, IPINFO_TOKEN_SETTING));
         } catch (GeneralSecurityException e) {
             // rethrow as a runtime exception, there's logging higher up the call chain around ReloadablePlugin
             throw new ElasticsearchException("Exception while reloading enterprise geoip download task executor", e);
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java
index dcaa8f6f2fb03..ae562d3c7359a 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java
@@ -11,7 +11,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -139,11 +138,18 @@ void updateDatabases() throws IOException {
         if (geoipIndex != null) {
             logger.trace("The {} index is not null", GeoIpDownloader.DATABASES_INDEX);
             if (clusterState.getRoutingTable().index(geoipIndex.getWriteIndex()).allPrimaryShardsActive() == false) {
-                throw new ElasticsearchException("not all primary shards of [" + DATABASES_INDEX + "] index are active");
+                logger.debug(
+                    "Not updating geoip database because not all primary shards of the [" + DATABASES_INDEX + "] index are active."
+                );
+                return;
             }
             var blockException = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, geoipIndex.getWriteIndex().getName());
             if (blockException != null) {
-                throw blockException;
+                logger.debug(
+                    "Not updating geoip database because there is a write block on the " + geoipIndex.getWriteIndex().getName() + " index",
+                    blockException
+                );
+                return;
             }
         }
         if (eagerDownloadSupplier.get() || atLeastOneGeoipProcessorSupplier.get()) {
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java
index a7828a9f3a0b7..61ca050d91c13 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java
@@ -43,18 +43,20 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteTransportException;
 
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
 
 import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX;
 import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER;
 import static org.elasticsearch.ingest.geoip.GeoIpProcessor.Factory.downloadDatabaseOnPipelineCreation;
 import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE;
+import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE;
 
 /**
  * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node.
@@ -237,14 +239,11 @@ public void clusterChanged(ClusterChangedEvent event) {
     }
 
     static boolean hasAtLeastOneGeoipProcessor(ClusterState clusterState) {
-        if (pipelineConfigurationsWithGeoIpProcessor(clusterState, true).isEmpty() == false) {
+        if (pipelinesWithGeoIpProcessor(clusterState, true).isEmpty() == false) {
             return true;
         }
 
-        Set<String> checkReferencedPipelines = pipelineConfigurationsWithGeoIpProcessor(clusterState, false).stream()
-            .map(PipelineConfiguration::getId)
-            .collect(Collectors.toSet());
-
+        final Set<String> checkReferencedPipelines = pipelinesWithGeoIpProcessor(clusterState, false);
         if (checkReferencedPipelines.isEmpty()) {
             return false;
         }
@@ -257,22 +256,24 @@ static boolean hasAtLeastOneGeoipProcessor(ClusterState clusterState) {
     }
 
     /**
-     * Retrieve list of pipelines that have at least one geoip processor.
+     * Retrieve the set of pipeline ids that have at least one geoip processor.
      * @param clusterState Cluster state.
      * @param downloadDatabaseOnPipelineCreation Filter the list to include only pipeline with the download_database_on_pipeline_creation
      *                                           matching the param.
-     * @return A list of {@link PipelineConfiguration} matching criteria.
+     * @return A set of pipeline ids matching criteria.
      */
     @SuppressWarnings("unchecked")
-    private static List<PipelineConfiguration> pipelineConfigurationsWithGeoIpProcessor(
-        ClusterState clusterState,
-        boolean downloadDatabaseOnPipelineCreation
-    ) {
-        List<PipelineConfiguration> pipelineDefinitions = IngestService.getPipelines(clusterState);
-        return pipelineDefinitions.stream().filter(pipelineConfig -> {
-            List<Map<String, Object>> processors = (List<Map<String, Object>>) pipelineConfig.getConfigAsMap().get(Pipeline.PROCESSORS_KEY);
-            return hasAtLeastOneGeoipProcessor(processors, downloadDatabaseOnPipelineCreation);
-        }).toList();
+    private static Set<String> pipelinesWithGeoIpProcessor(ClusterState clusterState, boolean downloadDatabaseOnPipelineCreation) {
+        List<PipelineConfiguration> configurations = IngestService.getPipelines(clusterState);
+        Set<String> ids = new HashSet<>();
+        // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+        for (PipelineConfiguration configuration : configurations) {
+            List<Map<String, Object>> processors = (List<Map<String, Object>>) configuration.getConfigAsMap().get(Pipeline.PROCESSORS_KEY);
+            if (hasAtLeastOneGeoipProcessor(processors, downloadDatabaseOnPipelineCreation)) {
+                ids.add(configuration.getId());
+            }
+        }
+        return Collections.unmodifiableSet(ids);
     }
 
     /**
@@ -282,7 +283,15 @@ private static List<PipelineConfiguration> pipelineConfigurationsWithGeoIpProces
      * @return true if a geoip processor is found in the processor list.
      */
     private static boolean hasAtLeastOneGeoipProcessor(List<Map<String, Object>> processors, boolean downloadDatabaseOnPipelineCreation) {
-        return processors != null && processors.stream().anyMatch(p -> hasAtLeastOneGeoipProcessor(p, downloadDatabaseOnPipelineCreation));
+        if (processors != null) {
+            // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+            for (Map<String, Object> processor : processors) {
+                if (hasAtLeastOneGeoipProcessor(processor, downloadDatabaseOnPipelineCreation)) {
+                    return true;
+                }
+            }
+        }
+        return false;
     }
 
     /**
@@ -297,9 +306,18 @@ private static boolean hasAtLeastOneGeoipProcessor(Map<String, Object> processor
             return false;
         }
 
-        final Map<String, Object> processorConfig = (Map<String, Object>) processor.get(GEOIP_TYPE);
-        if (processorConfig != null) {
-            return downloadDatabaseOnPipelineCreation(GEOIP_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation;
+        {
+            final Map<String, Object> processorConfig = (Map<String, Object>) processor.get(GEOIP_TYPE);
+            if (processorConfig != null) {
+                return downloadDatabaseOnPipelineCreation(processorConfig) == downloadDatabaseOnPipelineCreation;
+            }
+        }
+
+        {
+            final Map<String, Object> processorConfig = (Map<String, Object>) processor.get(IP_LOCATION_TYPE);
+            if (processorConfig != null) {
+                return downloadDatabaseOnPipelineCreation(processorConfig) == downloadDatabaseOnPipelineCreation;
+            }
         }
 
         return isProcessorWithOnFailureGeoIpProcessor(processor, downloadDatabaseOnPipelineCreation)
@@ -307,7 +325,7 @@ private static boolean hasAtLeastOneGeoipProcessor(Map<String, Object> processor
     }
 
     /**
-     * Check if a processor config is has an on_failure clause containing at least a geoip processor.
+     * Check if a processor config has an on_failure clause containing at least a geoip processor.
      * @param processor Processor config.
      * @param downloadDatabaseOnPipelineCreation Should the download_database_on_pipeline_creation of the geoip processor be true or false.
      * @return true if a geoip processor is found in the processor list.
@@ -317,16 +335,17 @@ private static boolean isProcessorWithOnFailureGeoIpProcessor(
         Map<String, Object> processor,
         boolean downloadDatabaseOnPipelineCreation
     ) {
-        return processor != null
-            && processor.values()
-                .stream()
-                .anyMatch(
-                    value -> value instanceof Map
-                        && hasAtLeastOneGeoipProcessor(
-                            ((Map<String, List<Map<String, Object>>>) value).get("on_failure"),
-                            downloadDatabaseOnPipelineCreation
-                        )
-                );
+        // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+        for (Object value : processor.values()) {
+            if (value instanceof Map
+                && hasAtLeastOneGeoipProcessor(
+                    ((Map<String, List<Map<String, Object>>>) value).get("on_failure"),
+                    downloadDatabaseOnPipelineCreation
+                )) {
+                return true;
+            }
+        }
+        return false;
     }
 
     /**
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index f8ca6d87924a4..f99f8dbe2fdd0 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -42,6 +42,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         + "in a future version of Elasticsearch"; // TODO add a message about migration?
 
     public static final String GEOIP_TYPE = "geoip";
+    public static final String IP_LOCATION_TYPE = "ip_location";
 
     private final String type;
     private final String field;
@@ -195,13 +196,19 @@ public IpDatabase get() throws IOException {
             }
 
             if (Assertions.ENABLED) {
-                // Only check whether the suffix has changed and not the entire database type.
-                // To sanity check whether a city db isn't overwriting with a country or asn db.
-                // For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different,
-                // by checking just the suffix this assertion doesn't fail.
-                String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-'));
-                assert loader.getDatabaseType().endsWith(expectedSuffix)
-                    : "database type [" + loader.getDatabaseType() + "] doesn't match with expected suffix [" + expectedSuffix + "]";
+                // Note that the expected suffix might be null for providers that aren't amenable to using dashes as separator for
+                // determining the database type.
+                int last = databaseType.lastIndexOf('-');
+                final String expectedSuffix = last == -1 ? null : databaseType.substring(last);
+
+                // If the entire database type matches, then that's a match. Otherwise, if there's a suffix to compare on, then
+                // check whether the suffix has changed (not the entire database type).
+                // This is to sanity check, for example, that a city db isn't overwritten with a country or asn db.
+                // But there are permissible overwrites that make sense, for example overwriting a geolite city db with a geoip city db
+                // is a valid change, but the db type is slightly different -- by checking just the suffix this assertion won't fail.
+                final String loaderType = loader.getDatabaseType();
+                assert loaderType.equals(databaseType) || expectedSuffix == null || loaderType.endsWith(expectedSuffix)
+                    : "database type [" + loaderType + "] doesn't match with expected suffix [" + expectedSuffix + "]";
             }
             return loader;
         }
@@ -225,15 +232,14 @@ public Processor create(
             final Map<String, Object> config
         ) throws IOException {
             String ipField = readStringProperty(type, processorTag, config, "field");
-            String targetField = readStringProperty(type, processorTag, config, "target_field", "geoip");
+            String targetField = readStringProperty(type, processorTag, config, "target_field", type);
             String databaseFile = readStringProperty(type, processorTag, config, "database_file", "GeoLite2-City.mmdb");
             List<String> propertyNames = readOptionalList(type, processorTag, config, "properties");
             boolean ignoreMissing = readBooleanProperty(type, processorTag, config, "ignore_missing", false);
             boolean firstOnly = readBooleanProperty(type, processorTag, config, "first_only", true);
 
-            // Validating the download_database_on_pipeline_creation even if the result
-            // is not used directly by the factory.
-            downloadDatabaseOnPipelineCreation(type, config, processorTag);
+            // validate (and consume) the download_database_on_pipeline_creation property even though the result is not used by the factory
+            readBooleanProperty(type, processorTag, config, "download_database_on_pipeline_creation", true);
 
             // noop, should be removed in 9.0
             Object value = config.remove("fallback_to_default_databases");
@@ -312,8 +318,15 @@ public Processor create(
             );
         }
 
-        public static boolean downloadDatabaseOnPipelineCreation(String type, Map<String, Object> config, String processorTag) {
-            return readBooleanProperty(type, processorTag, config, "download_database_on_pipeline_creation", true);
+        /**
+         * Get the value of the "download_database_on_pipeline_creation" property from a processor's config map.
+         * <p>
+         * As with the actual property definition, the default value of the property is 'true'. Unlike the actual
+         * property definition, this method doesn't consume (that is, <code>config.remove</code>) the property from
+         * the config map.
+         */
+        public static boolean downloadDatabaseOnPipelineCreation(Map<String, Object> config) {
+            return (boolean) config.getOrDefault("download_database_on_pipeline_creation", true);
         }
     }
 
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java
index 09ed10568ce8d..47ca79e3cb3b9 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java
@@ -44,8 +44,7 @@
 public class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable {
 
     private static boolean includeSha256(TransportVersion version) {
-        return version.isPatchFrom(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15)
-            || version.onOrAfter(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER);
+        return version.isPatchFrom(TransportVersions.V_8_15_0) || version.onOrAfter(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER);
     }
 
     private static final ParseField DATABASES = new ParseField("databases");
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 49932f342086e..3107f0bed55e8 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -71,6 +71,7 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
+import static java.util.Map.entry;
 import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
 import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER;
 import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN;
@@ -111,7 +112,8 @@ public List<Setting<?>> getSettings() {
             GeoIpDownloaderTaskExecutor.ENABLED_SETTING,
             GeoIpDownloader.ENDPOINT_SETTING,
             GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING,
-            EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING
+            EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING,
+            EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING
         );
     }
 
@@ -129,7 +131,10 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
             parameters.ingestService.getClusterService()
         );
         databaseRegistry.set(registry);
-        return Map.of(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry));
+        return Map.ofEntries(
+            entry(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry)),
+            entry(GeoIpProcessor.IP_LOCATION_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.IP_LOCATION_TYPE, registry))
+        );
     }
 
     @Override
@@ -239,6 +244,11 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 DatabaseConfiguration.Maxmind.NAME,
                 DatabaseConfiguration.Maxmind::new
             ),
+            new NamedWriteableRegistry.Entry(
+                DatabaseConfiguration.Provider.class,
+                DatabaseConfiguration.Ipinfo.NAME,
+                DatabaseConfiguration.Ipinfo::new
+            ),
             new NamedWriteableRegistry.Entry(
                 DatabaseConfiguration.Provider.class,
                 DatabaseConfiguration.Local.NAME,
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java
index 3379fdff0633a..e879f0e0e3514 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java
@@ -13,9 +13,16 @@
 import org.elasticsearch.core.Nullable;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.IPINFO_PREFIX;
+import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoDatabase;
+import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoLookup;
+import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindDatabase;
+import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindLookup;
+
 final class IpDataLookupFactories {
 
     private IpDataLookupFactories() {
@@ -26,78 +33,44 @@ interface IpDataLookupFactory {
         IpDataLookup create(List<String> properties);
     }
 
-    private static final String CITY_DB_SUFFIX = "-City";
-    private static final String COUNTRY_DB_SUFFIX = "-Country";
-    private static final String ASN_DB_SUFFIX = "-ASN";
-    private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
-    private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type";
-    private static final String DOMAIN_DB_SUFFIX = "-Domain";
-    private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
-    private static final String ISP_DB_SUFFIX = "-ISP";
-
-    @Nullable
-    private static Database getMaxmindDatabase(final String databaseType) {
-        if (databaseType.endsWith(CITY_DB_SUFFIX)) {
-            return Database.City;
-        } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
-            return Database.Country;
-        } else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
-            return Database.Asn;
-        } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) {
-            return Database.AnonymousIp;
-        } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) {
-            return Database.ConnectionType;
-        } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) {
-            return Database.Domain;
-        } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) {
-            return Database.Enterprise;
-        } else if (databaseType.endsWith(ISP_DB_SUFFIX)) {
-            return Database.Isp;
-        } else {
-            return null; // no match was found
-        }
-    }
-
     /**
      * Parses the passed-in databaseType and return the Database instance that is
      * associated with that databaseType.
      *
      * @param databaseType the database type String from the metadata of the database file
-     * @return the Database instance that is associated with the databaseType
+     * @return the Database instance that is associated with the databaseType (or null)
      */
     @Nullable
     static Database getDatabase(final String databaseType) {
         Database database = null;
 
         if (Strings.hasText(databaseType)) {
-            database = getMaxmindDatabase(databaseType);
+            final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT);
+            if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) {
+                database = getIpinfoDatabase(databaseTypeLowerCase); // all lower case!
+            } else {
+                // for historical reasons, fall back to assuming maxmind-like type parsing
+                database = getMaxmindDatabase(databaseType);
+            }
         }
 
         return database;
     }
 
-    @Nullable
-    static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Database database) {
-        return switch (database) {
-            case City -> MaxmindIpDataLookups.City::new;
-            case Country -> MaxmindIpDataLookups.Country::new;
-            case Asn -> MaxmindIpDataLookups.Asn::new;
-            case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new;
-            case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new;
-            case Domain -> MaxmindIpDataLookups.Domain::new;
-            case Enterprise -> MaxmindIpDataLookups.Enterprise::new;
-            case Isp -> MaxmindIpDataLookups.Isp::new;
-            default -> null;
-        };
-    }
-
     static IpDataLookupFactory get(final String databaseType, final String databaseFile) {
         final Database database = getDatabase(databaseType);
         if (database == null) {
             throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]");
         }
 
-        final Function<Set<Database.Property>, IpDataLookup> factoryMethod = getMaxmindLookup(database);
+        final Function<Set<Database.Property>, IpDataLookup> factoryMethod;
+        final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT);
+        if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) {
+            factoryMethod = getIpinfoLookup(database);
+        } else {
+            // for historical reasons, fall back to assuming maxmind-like types
+            factoryMethod = getMaxmindLookup(database);
+        }
 
         if (factoryMethod == null) {
             throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]");
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
index efc6734b3bd93..8ce2424844d9d 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
@@ -23,10 +23,14 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * A collection of {@link IpDataLookup} implementations for IPinfo databases
@@ -43,6 +47,81 @@ private IpinfoIpDataLookups() {
     // prefix dispatch and checks case-insensitive, so that works out nicely
     static final String IPINFO_PREFIX = "ipinfo";
 
+    private static final Set<String> IPINFO_TYPE_STOP_WORDS = Set.of(
+        "ipinfo",
+        "extended",
+        "free",
+        "generic",
+        "ip",
+        "sample",
+        "standard",
+        "mmdb"
+    );
+
+    /**
+     * Cleans up the database_type String from an ipinfo database by splitting on punctuation, removing stop words, and then joining
+     * with an underscore.
+     * <p>
+     * e.g. "ipinfo free_foo_sample.mmdb" -> "foo"
+     *
+     * @param type the database_type from an ipinfo database
+     * @return a cleaned up database_type string
+     */
+    // n.b. this is just based on observation of the types from a survey of such databases -- it's like browser user agent sniffing,
+    // there aren't necessarily any amazing guarantees about this behavior
+    static String ipinfoTypeCleanup(String type) {
+        List<String> parts = Arrays.asList(type.split("[ _.]"));
+        return parts.stream().filter((s) -> IPINFO_TYPE_STOP_WORDS.contains(s) == false).collect(Collectors.joining("_"));
+    }
+
+    @Nullable
+    static Database getIpinfoDatabase(final String databaseType) {
+        // for ipinfo the database selection is more along the lines of user-agent sniffing than
+        // string-based dispatch. the specific database_type strings could change in the future,
+        // hence the somewhat loose nature of this checking.
+
+        final String cleanedType = ipinfoTypeCleanup(databaseType);
+
+        // early detection on any of the 'extended' types
+        if (databaseType.contains("extended")) {
+            // which are not currently supported
+            logger.trace("returning null for unsupported database_type [{}]", databaseType);
+            return null;
+        }
+
+        // early detection on 'country_asn' so the 'country' and 'asn' checks don't get faked out
+        if (cleanedType.contains("country_asn")) {
+            // but it's not currently supported
+            logger.trace("returning null for unsupported database_type [{}]", databaseType);
+            return null;
+        }
+
+        if (cleanedType.contains("asn")) {
+            return Database.AsnV2;
+        } else if (cleanedType.contains("country")) {
+            return Database.CountryV2;
+        } else if (cleanedType.contains("location")) { // note: catches 'location' and 'geolocation' ;)
+            return Database.CityV2;
+        } else if (cleanedType.contains("privacy")) {
+            return Database.PrivacyDetection;
+        } else {
+            // no match was found
+            logger.trace("returning null for unsupported database_type [{}]", databaseType);
+            return null;
+        }
+    }
+
+    @Nullable
+    static Function<Set<Database.Property>, IpDataLookup> getIpinfoLookup(final Database database) {
+        return switch (database) {
+            case AsnV2 -> IpinfoIpDataLookups.Asn::new;
+            case CountryV2 -> IpinfoIpDataLookups.Country::new;
+            case CityV2 -> IpinfoIpDataLookups.Geolocation::new;
+            case PrivacyDetection -> IpinfoIpDataLookups.PrivacyDetection::new;
+            default -> null;
+        };
+    }
+
     /**
      * Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible).
      * @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long
@@ -139,8 +218,8 @@ public record CountryResult(
     public record GeolocationResult(
         String city,
         String country,
-        Double latitude,
-        Double longitude,
+        Double lat,
+        Double lng,
         String postalCode,
         String region,
         String timezone
@@ -150,14 +229,15 @@ public record GeolocationResult(
         public GeolocationResult(
             @MaxMindDbParameter(name = "city") String city,
             @MaxMindDbParameter(name = "country") String country,
-            @MaxMindDbParameter(name = "latitude") String latitude,
-            @MaxMindDbParameter(name = "longitude") String longitude,
-            // @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this
+            // @MaxMindDbParameter(name = "geoname_id") String geonameId, // for now we're not exposing this
+            @MaxMindDbParameter(name = "lat") String lat,
+            @MaxMindDbParameter(name = "lng") String lng,
             @MaxMindDbParameter(name = "postal_code") String postalCode,
             @MaxMindDbParameter(name = "region") String region,
+            // @MaxMindDbParameter(name = "region_code") String regionCode, // for now we're not exposing this
             @MaxMindDbParameter(name = "timezone") String timezone
         ) {
-            this(city, country, parseLocationDouble(latitude), parseLocationDouble(longitude), postalCode, region, timezone);
+            this(city, country, parseLocationDouble(lat), parseLocationDouble(lng), postalCode, region, timezone);
         }
     }
 
@@ -316,8 +396,8 @@ protected Map<String, Object> transform(final Result<GeolocationResult> result)
                         }
                     }
                     case LOCATION -> {
-                        Double latitude = response.latitude;
-                        Double longitude = response.longitude;
+                        Double latitude = response.lat;
+                        Double longitude = response.lng;
                         if (latitude != null && longitude != null) {
                             Map<String, Object> locationObject = new HashMap<>();
                             locationObject.put("lat", latitude);
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java
index 4297413073e52..8bc74c0e4aac4 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java
@@ -26,6 +26,8 @@
 import com.maxmind.geoip2.record.Postal;
 import com.maxmind.geoip2.record.Subdivision;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.core.Nullable;
@@ -37,6 +39,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * A collection of {@link IpDataLookup} implementations for MaxMind databases
@@ -47,11 +50,63 @@ private MaxmindIpDataLookups() {
         // utility class
     }
 
+    private static final Logger logger = LogManager.getLogger(MaxmindIpDataLookups.class);
+
     // the actual prefixes from the metadata are cased like the literal strings, but
     // prefix dispatch and checks case-insensitive, so the actual constants are lowercase
     static final String GEOIP2_PREFIX = "GeoIP2".toLowerCase(Locale.ROOT);
     static final String GEOLITE2_PREFIX = "GeoLite2".toLowerCase(Locale.ROOT);
 
+    // note: the secondary dispatch on suffix happens to be case sensitive
+    private static final String CITY_DB_SUFFIX = "-City";
+    private static final String COUNTRY_DB_SUFFIX = "-Country";
+    private static final String ASN_DB_SUFFIX = "-ASN";
+    private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
+    private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type";
+    private static final String DOMAIN_DB_SUFFIX = "-Domain";
+    private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
+    private static final String ISP_DB_SUFFIX = "-ISP";
+
+    @Nullable
+    static Database getMaxmindDatabase(final String databaseType) {
+        if (databaseType.endsWith(CITY_DB_SUFFIX)) {
+            return Database.City;
+        } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
+            return Database.Country;
+        } else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
+            return Database.Asn;
+        } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) {
+            return Database.AnonymousIp;
+        } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) {
+            return Database.ConnectionType;
+        } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) {
+            return Database.Domain;
+        } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) {
+            return Database.Enterprise;
+        } else if (databaseType.endsWith(ISP_DB_SUFFIX)) {
+            return Database.Isp;
+        } else {
+            // no match was found
+            logger.trace("returning null for unsupported database_type [{}]", databaseType);
+            return null;
+        }
+    }
+
+    @Nullable
+    static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Database database) {
+        return switch (database) {
+            case City -> MaxmindIpDataLookups.City::new;
+            case Country -> MaxmindIpDataLookups.Country::new;
+            case Asn -> MaxmindIpDataLookups.Asn::new;
+            case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new;
+            case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new;
+            case Domain -> MaxmindIpDataLookups.Domain::new;
+            case Enterprise -> MaxmindIpDataLookups.Enterprise::new;
+            case Isp -> MaxmindIpDataLookups.Isp::new;
+            default -> null;
+        };
+    }
+
     static class AnonymousIp extends AbstractBase<AnonymousIpResponse> {
         AnonymousIp(final Set<Database.Property> properties) {
             super(
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java
index 3399b71879e26..a26364f9305e1 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java
@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Objects;
 import java.util.Set;
 import java.util.regex.Pattern;
@@ -78,8 +79,19 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i
         // "GeoLite2-Country"
     );
 
+    public static final Set<String> IPINFO_NAMES = Set.of(
+        // these file names are from https://ipinfo.io/developers/database-filename-reference
+        "asn", // "Free IP to ASN"
+        "country", // "Free IP to Country"
+        // "country_asn" // "Free IP to Country + IP to ASN", not supported at present
+        "standard_asn", // commercial "ASN"
+        "standard_location", // commercial "IP Geolocation"
+        "standard_privacy" // commercial "Privacy Detection" (sometimes "Anonymous IP")
+    );
+
     private static final ParseField NAME = new ParseField("name");
     private static final ParseField MAXMIND = new ParseField(Maxmind.NAME);
+    private static final ParseField IPINFO = new ParseField(Ipinfo.NAME);
     private static final ParseField WEB = new ParseField(Web.NAME);
     private static final ParseField LOCAL = new ParseField(Local.NAME);
 
@@ -89,12 +101,21 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i
         (a, id) -> {
             String name = (String) a[0];
             Provider provider;
+
+            // one and only one provider object must be present
+            final long numNonNulls = Arrays.stream(a, 1, a.length).filter(Objects::nonNull).count();
+            if (numNonNulls != 1) {
+                throw new IllegalArgumentException("Exactly one provider object must be specified, but [" + numNonNulls + "] were found");
+            }
+
             if (a[1] != null) {
                 provider = (Maxmind) a[1];
             } else if (a[2] != null) {
-                provider = (Web) a[2];
+                provider = (Ipinfo) a[2];
+            } else if (a[3] != null) {
+                provider = (Web) a[3];
             } else {
-                provider = (Local) a[3];
+                provider = (Local) a[4];
             }
             return new DatabaseConfiguration(id, name, provider);
         }
@@ -107,6 +128,7 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i
             (parser, id) -> Maxmind.PARSER.apply(parser, null),
             MAXMIND
         );
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Ipinfo.PARSER.apply(parser, null), IPINFO);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Web.PARSER.apply(parser, null), WEB);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Local.PARSER.apply(parser, null), LOCAL);
     }
@@ -194,8 +216,16 @@ public ActionRequestValidationException validate() {
             err.addValidationError("invalid name [" + name + "]: cannot be empty");
         }
 
-        if (MAXMIND_NAMES.contains(name) == false) {
-            err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])");
+        // provider-specific name validation
+        if (provider instanceof Maxmind) {
+            if (MAXMIND_NAMES.contains(name) == false) {
+                err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])");
+            }
+        }
+        if (provider instanceof Ipinfo) {
+            if (IPINFO_NAMES.contains(name) == false) {
+                err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + IPINFO_NAMES + "])");
+            }
         }
 
         // important: the name must be unique across all configurations of this same type,
@@ -234,7 +264,7 @@ public String getWriteableName() {
 
         private static final ParseField ACCOUNT_ID = new ParseField("account_id");
 
-        private static final ConstructingObjectParser<Maxmind, Void> PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> {
+        private static final ConstructingObjectParser<Maxmind, Void> PARSER = new ConstructingObjectParser<>("maxmind", false, (a, id) -> {
             String accountId = (String) a[0];
             return new Maxmind(accountId);
         });
@@ -247,10 +277,6 @@ public Maxmind(StreamInput in) throws IOException {
             this(in.readString());
         }
 
-        public static Maxmind parse(XContentParser parser) {
-            return PARSER.apply(parser, null);
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(accountId);
@@ -270,6 +296,37 @@ public boolean isReadOnly() {
         }
     }
 
+    public record Ipinfo() implements Provider {
+        public static final String NAME = "ipinfo";
+
+        // this'll become a ConstructingObjectParser once we accept the token (securely) in the json definition
+        private static final ObjectParser<Ipinfo, Void> PARSER = new ObjectParser<>("ipinfo", Ipinfo::new);
+
+        public Ipinfo(StreamInput in) throws IOException {
+            this();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {}
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public boolean isReadOnly() {
+            return false;
+        }
+    }
+
     public record Local(String type) implements Provider {
         public static final String NAME = "local";
 
@@ -288,10 +345,6 @@ public Local(StreamInput in) throws IOException {
             this(in.readString());
         }
 
-        public static Local parse(XContentParser parser) {
-            return PARSER.apply(parser, null);
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(type);
@@ -325,10 +378,6 @@ public Web(StreamInput in) throws IOException {
             this();
         }
 
-        public static Web parse(XContentParser parser) {
-            return PARSER.apply(parser, null);
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {}
 
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java
index 82888fa39c857..fcfd8e51aabb5 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java
@@ -66,7 +66,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         // (we'll be a in a json map where the id is the key)
         builder.startObject();
         builder.field(VERSION.getPreferredName(), version);
-        builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate);
+        builder.timestampFieldsFromUnixEpochMillis(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate);
         builder.field(DATABASE.getPreferredName(), database);
         builder.endObject();
         return builder;
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java
index 89bc3d1ce5d37..1970883e91b3e 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java
@@ -110,7 +110,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.startObject();
                 builder.field("id", database.id()); // serialize including the id -- this is get response serialization
                 builder.field(VERSION.getPreferredName(), item.version());
-                builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), item.modifiedDate());
+                builder.timestampFieldsFromUnixEpochMillis(
+                    MODIFIED_DATE_MILLIS.getPreferredName(),
+                    MODIFIED_DATE.getPreferredName(),
+                    item.modifiedDate()
+                );
                 builder.field(DATABASE.getPreferredName(), database);
                 builder.endObject();
             }
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java
index e836821a3b2f2..78ea73250d632 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java
@@ -27,7 +27,7 @@ public class RestDeleteDatabaseConfigurationAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(new Route(DELETE, "/_ingest/geoip/database/{id}"));
+        return List.of(new Route(DELETE, "/_ingest/ip_location/database/{id}"), new Route(DELETE, "/_ingest/geoip/database/{id}"));
     }
 
     @Override
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java
index f34f388f22965..af446ee8d2bd9 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java
@@ -26,7 +26,12 @@ public class RestGetDatabaseConfigurationAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(new Route(GET, "/_ingest/geoip/database"), new Route(GET, "/_ingest/geoip/database/{id}"));
+        return List.of(
+            new Route(GET, "/_ingest/ip_location/database"),
+            new Route(GET, "/_ingest/ip_location/database/{id}"),
+            new Route(GET, "/_ingest/geoip/database"),
+            new Route(GET, "/_ingest/geoip/database/{id}")
+        );
     }
 
     @Override
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java
index c0b7a3f59f3aa..95b40df12fd1f 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java
@@ -29,7 +29,7 @@ public class RestPutDatabaseConfigurationAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(new Route(PUT, "/_ingest/geoip/database/{id}"));
+        return List.of(new Route(PUT, "/_ingest/ip_location/database/{id}"), new Route(PUT, "/_ingest/geoip/database/{id}"));
     }
 
     @Override
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
index fda0e12bb1b76..dfb8fa78089d2 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
 import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request;
 import org.elasticsearch.injection.guice.Inject;
@@ -41,6 +42,8 @@
 import java.util.Map;
 import java.util.Optional;
 
+import static org.elasticsearch.ingest.IngestGeoIpFeatures.PUT_DATABASE_CONFIGURATION_ACTION_IPINFO;
+
 public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class);
@@ -58,6 +61,7 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) {
         }
     };
 
+    private final FeatureService featureService;
     private final MasterServiceTaskQueue<UpdateDatabaseConfigurationTask> updateDatabaseConfigurationTaskQueue;
 
     @Inject
@@ -66,7 +70,8 @@ public TransportPutDatabaseConfigurationAction(
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        FeatureService featureService
     ) {
         super(
             PutDatabaseConfigurationAction.NAME,
@@ -79,6 +84,7 @@ public TransportPutDatabaseConfigurationAction(
             AcknowledgedResponse::readFrom,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
+        this.featureService = featureService;
         this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue(
             "update-geoip-database-configuration-state-update",
             Priority.NORMAL,
@@ -89,6 +95,19 @@ public TransportPutDatabaseConfigurationAction(
     @Override
     protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
         final String id = request.getDatabase().id();
+
+        // if this is an ipinfo configuration, then make sure the whole cluster supports that feature
+        if (request.getDatabase().provider() instanceof DatabaseConfiguration.Ipinfo
+            && featureService.clusterHasFeature(clusterService.state(), PUT_DATABASE_CONFIGURATION_ACTION_IPINFO) == false) {
+            listener.onFailure(
+                new IllegalArgumentException(
+                    "Unable to use ipinfo database configurations in mixed-clusters with nodes that do not support feature "
+                        + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO.id()
+                )
+            );
+            return;
+        }
+
         updateDatabaseConfigurationTaskQueue.submitTask(
             Strings.format("update-geoip-database-configuration-[%s]", id),
             new UpdateDatabaseConfigurationTask(listener, request.getDatabase()),
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java
new file mode 100644
index 0000000000000..39ecf4e70383b
--- /dev/null
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.ingest.geoip;
+
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Set;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class DatabaseTests extends ESTestCase {
+
+    public void testDatabasePropertyInvariants() {
+        // the city database is like a specialization of the country database
+        assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty()));
+        assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty()));
+
+        // the isp database is like a specialization of the asn database
+        assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty()));
+        assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty()));
+
+        // the enterprise database is like these other databases joined together
+        for (Database type : Set.of(
+            Database.City,
+            Database.Country,
+            Database.Asn,
+            Database.AnonymousIp,
+            Database.ConnectionType,
+            Database.Domain,
+            Database.Isp
+        )) {
+            assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty()));
+        }
+        // but in terms of the default fields, it's like a drop-in replacement for the city database
+        // n.b. this is just a choice we decided to make here at Elastic
+        assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties()));
+    }
+
+    public void testDatabaseVariantPropertyInvariants() {
+        // the second ASN variant database is like a specialization of the ASN database
+        assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty()));
+        assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties()));
+
+        // the second City variant database is like a version of the ordinary City database but lacking many fields
+        assertThat(Sets.difference(Database.CityV2.properties(), Database.City.properties()), is(empty()));
+        assertThat(Sets.difference(Database.CityV2.defaultProperties(), Database.City.defaultProperties()), is(empty()));
+
+        // the second Country variant database is like a version of the ordinary Country database but lacking come fields
+        assertThat(Sets.difference(Database.CountryV2.properties(), Database.CountryV2.properties()), is(empty()));
+        assertThat(Database.CountryV2.defaultProperties(), equalTo(Database.Country.defaultProperties()));
+    }
+}
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java
index 88c37409713ac..e1cd127be9c87 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java
@@ -488,6 +488,36 @@ public void testMaxmindUrls() {
         }
     }
 
+    public void testIpinfoUrls() {
+        // a 'free' database like 'asn' has 'free/' in the url (automatically)
+        final EnterpriseGeoIpDownloader.IpinfoDownload download = geoIpDownloader.new IpinfoDownload(
+            "asn", new DatabaseConfiguration.Ipinfo()
+        );
+
+        {
+            String url = "https://ipinfo.io/data/free/asn.mmdb";
+            assertThat(download.url("mmdb"), equalTo(url));
+        }
+        {
+            String url = "https://ipinfo.io/data/free/asn.mmdb/checksums";
+            assertThat(download.url("mmdb/checksums"), equalTo(url));
+        }
+
+        // but a non-'free' database like 'standard_asn' does not
+        final EnterpriseGeoIpDownloader.IpinfoDownload download2 = geoIpDownloader.new IpinfoDownload(
+            "standard_asn", new DatabaseConfiguration.Ipinfo()
+        );
+
+        {
+            String url = "https://ipinfo.io/data/standard_asn.mmdb";
+            assertThat(download2.url("mmdb"), equalTo(url));
+        }
+        {
+            String url = "https://ipinfo.io/data/standard_asn.mmdb/checksums";
+            assertThat(download2.url("mmdb/checksums"), equalTo(url));
+        }
+    }
+
     private static class MockClient extends NoOpClient {
 
         private final Map<ActionType<?>, BiConsumer<? extends ActionRequest, ? extends ActionListener<?>>> handlers = new HashMap<>();
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
index e73f0a36cc632..5698328792787 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.ingest.geoip;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -25,11 +24,9 @@
 import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -583,37 +580,28 @@ void processDatabase(Map<String, Object> databaseInfo) {
         assertFalse(it.hasNext());
     }
 
-    public void testUpdateDatabasesWriteBlock() {
+    public void testUpdateDatabasesWriteBlock() throws IOException {
+        /*
+         * Here we make sure that we bail out before making an httpClient request if there is write block on the .geoip_databases index
+         */
         ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of()));
         var geoIpIndex = state.getMetadata().getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName();
         state = ClusterState.builder(state)
             .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
             .build();
         when(clusterService.state()).thenReturn(state);
-        var e = expectThrows(ClusterBlockException.class, () -> geoIpDownloader.updateDatabases());
-        assertThat(
-            e.getMessage(),
-            equalTo(
-                "index ["
-                    + geoIpIndex
-                    + "] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, "
-                    + "index has read-only-allow-delete block; for more information, see "
-                    + ReferenceDocs.FLOOD_STAGE_WATERMARK
-                    + "];"
-            )
-        );
+        geoIpDownloader.updateDatabases();
         verifyNoInteractions(httpClient);
     }
 
-    public void testUpdateDatabasesIndexNotReady() {
+    public void testUpdateDatabasesIndexNotReady() throws IOException {
+        /*
+         * Here we make sure that we bail out before making an httpClient request if there are unallocated shards on the .geoip_databases
+         * index
+         */
         ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of()), true);
-        var geoIpIndex = state.getMetadata().getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName();
-        state = ClusterState.builder(state)
-            .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
-            .build();
         when(clusterService.state()).thenReturn(state);
-        var e = expectThrows(ElasticsearchException.class, () -> geoIpDownloader.updateDatabases());
-        assertThat(e.getMessage(), equalTo("not all primary shards of [.geoip_databases] index are active"));
+        geoIpDownloader.updateDatabases();
         verifyNoInteractions(httpClient);
     }
 
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index 50b59c26749fc..4548e92239ce1 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.ingest.geoip;
 
 import org.elasticsearch.common.CheckedSupplier;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
@@ -24,14 +23,13 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE;
+import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE;
 import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
@@ -40,10 +38,6 @@
 
 public class GeoIpProcessorTests extends ESTestCase {
 
-    private static IpDataLookup ipDataLookupAll(final Database database) {
-        return IpDataLookupFactories.getMaxmindLookup(database).apply(database.properties());
-    }
-
     // a temporary directory that mmdb files can be copied to and read from
     private Path tmpDir;
 
@@ -57,43 +51,47 @@ public void cleanup() throws IOException {
         IOUtils.rm(tmpDir);
     }
 
-    public void testDatabasePropertyInvariants() {
-        // the city database is like a specialization of the country database
-        assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty()));
-        assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty()));
-
-        // the isp database is like a specialization of the asn database
-        assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty()));
-        assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty()));
-
-        // the enterprise database is like these other databases joined together
-        for (Database type : Set.of(
-            Database.City,
-            Database.Country,
-            Database.Asn,
-            Database.AnonymousIp,
-            Database.ConnectionType,
-            Database.Domain,
-            Database.Isp
-        )) {
-            assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty()));
-        }
-        // but in terms of the default fields, it's like a drop-in replacement for the city database
-        // n.b. this is just a choice we decided to make here at Elastic
-        assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties()));
+    public void testMaxmindCity() throws Exception {
+        String ip = "2602:306:33d3:8000::3257:9652";
+        GeoIpProcessor processor = new GeoIpProcessor(
+            GEOIP_TYPE, // n.b. this is a "geoip" processor
+            randomAlphaOfLength(10),
+            null,
+            "source_field",
+            loader("GeoLite2-City.mmdb"),
+            () -> true,
+            "target_field",
+            getMaxmindCityLookup(),
+            false,
+            false,
+            "filename"
+        );
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field", ip);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
+        @SuppressWarnings("unchecked")
+        Map<String, Object> data = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(data, notNullValue());
+        assertThat(data.get("ip"), equalTo(ip));
+        assertThat(data.get("city_name"), equalTo("Homestead"));
+        // see MaxmindIpDataLookupsTests for more tests of the data lookup behavior
     }
 
-    public void testCity() throws Exception {
-        String ip = "8.8.8.8";
+    public void testIpinfoGeolocation() throws Exception {
+        String ip = "72.20.12.220";
         GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
+            IP_LOCATION_TYPE, // n.b. this is an "ip_location" processor
             randomAlphaOfLength(10),
             null,
             "source_field",
-            loader("GeoLite2-City.mmdb"),
+            loader("ipinfo/ip_geolocation_standard_sample.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getIpinfoGeolocationLookup(),
             false,
             false,
             "filename"
@@ -106,20 +104,11 @@ public void testCity() throws Exception {
 
         assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
         @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(12));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("country_in_european_union"), equalTo(false));
-        assertThat(geoData.get("country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("country_name"), equalTo("United States"));
-        assertThat(geoData.get("continent_code"), equalTo("NA"));
-        assertThat(geoData.get("continent_name"), equalTo("North America"));
-        assertThat(geoData.get("timezone"), equalTo("America/Chicago"));
-        assertThat(geoData.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
-        assertThat(geoData.get("registered_country_in_european_union"), equalTo(false));
-        assertThat(geoData.get("registered_country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("registered_country_name"), equalTo("United States"));
+        Map<String, Object> data = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(data, notNullValue());
+        assertThat(data.get("ip"), equalTo(ip));
+        assertThat(data.get("city_name"), equalTo("Chicago"));
+        // see IpinfoIpDataLookupsTests for more tests of the data lookup behavior
     }
 
     public void testNullValueWithIgnoreMissing() throws Exception {
@@ -131,7 +120,7 @@ public void testNullValueWithIgnoreMissing() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             true,
             false,
             "filename"
@@ -154,7 +143,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             true,
             false,
             "filename"
@@ -174,7 +163,7 @@ public void testNullWithoutIgnoreMissing() {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "filename"
@@ -197,7 +186,7 @@ public void testNonExistentWithoutIgnoreMissing() {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "filename"
@@ -208,369 +197,6 @@ public void testNonExistentWithoutIgnoreMissing() {
         assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]"));
     }
 
-    public void testCity_withIpV6() throws Exception {
-        String ip = "2602:306:33d3:8000::3257:9652";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoLite2-City.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.City),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(16));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("country_in_european_union"), equalTo(false));
-        assertThat(geoData.get("country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("country_name"), equalTo("United States"));
-        assertThat(geoData.get("continent_code"), equalTo("NA"));
-        assertThat(geoData.get("continent_name"), equalTo("North America"));
-        assertThat(geoData.get("region_iso_code"), equalTo("US-FL"));
-        assertThat(geoData.get("region_name"), equalTo("Florida"));
-        assertThat(geoData.get("city_name"), equalTo("Homestead"));
-        assertThat(geoData.get("timezone"), equalTo("America/New_York"));
-        assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d)));
-        assertThat(geoData.get("accuracy_radius"), equalTo(50));
-        assertThat(geoData.get("postal_code"), equalTo("33035"));
-        assertThat(geoData.get("registered_country_in_european_union"), equalTo(false));
-        assertThat(geoData.get("registered_country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("registered_country_name"), equalTo("United States"));
-    }
-
-    public void testCityWithMissingLocation() throws Exception {
-        String ip = "80.231.5.0";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoLite2-City.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.City),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(1));
-        assertThat(geoData.get("ip"), equalTo(ip));
-    }
-
-    public void testCountry() throws Exception {
-        String ip = "82.170.213.79";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoLite2-Country.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.Country),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(9));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("country_in_european_union"), equalTo(true));
-        assertThat(geoData.get("country_iso_code"), equalTo("NL"));
-        assertThat(geoData.get("country_name"), equalTo("Netherlands"));
-        assertThat(geoData.get("continent_code"), equalTo("EU"));
-        assertThat(geoData.get("continent_name"), equalTo("Europe"));
-        assertThat(geoData.get("registered_country_in_european_union"), equalTo(true));
-        assertThat(geoData.get("registered_country_iso_code"), equalTo("NL"));
-        assertThat(geoData.get("registered_country_name"), equalTo("Netherlands"));
-    }
-
-    public void testCountryWithMissingLocation() throws Exception {
-        String ip = "80.231.5.0";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoLite2-Country.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.Country),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(1));
-        assertThat(geoData.get("ip"), equalTo(ip));
-    }
-
-    public void testAsn() throws Exception {
-        String ip = "82.171.64.0";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoLite2-ASN.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.Asn),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(4));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("asn"), equalTo(1136L));
-        assertThat(geoData.get("organization_name"), equalTo("KPN B.V."));
-        assertThat(geoData.get("network"), equalTo("82.168.0.0/14"));
-    }
-
-    public void testAnonymmousIp() throws Exception {
-        String ip = "81.2.69.1";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoIP2-Anonymous-IP-Test.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.AnonymousIp),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(7));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("hosting_provider"), equalTo(true));
-        assertThat(geoData.get("tor_exit_node"), equalTo(true));
-        assertThat(geoData.get("anonymous_vpn"), equalTo(true));
-        assertThat(geoData.get("anonymous"), equalTo(true));
-        assertThat(geoData.get("public_proxy"), equalTo(true));
-        assertThat(geoData.get("residential_proxy"), equalTo(true));
-    }
-
-    public void testConnectionType() throws Exception {
-        String ip = "214.78.120.5";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoIP2-Connection-Type-Test.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.ConnectionType),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(2));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("connection_type"), equalTo("Satellite"));
-    }
-
-    public void testDomain() throws Exception {
-        String ip = "69.219.64.2";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoIP2-Domain-Test.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.Domain),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(2));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("domain"), equalTo("ameritech.net"));
-    }
-
-    public void testEnterprise() throws Exception {
-        String ip = "74.209.24.4";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoIP2-Enterprise-Test.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.Enterprise),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(33));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("country_confidence"), equalTo(99));
-        assertThat(geoData.get("country_in_european_union"), equalTo(false));
-        assertThat(geoData.get("country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("country_name"), equalTo("United States"));
-        assertThat(geoData.get("continent_code"), equalTo("NA"));
-        assertThat(geoData.get("continent_name"), equalTo("North America"));
-        assertThat(geoData.get("region_iso_code"), equalTo("US-NY"));
-        assertThat(geoData.get("region_name"), equalTo("New York"));
-        assertThat(geoData.get("city_confidence"), equalTo(11));
-        assertThat(geoData.get("city_name"), equalTo("Chatham"));
-        assertThat(geoData.get("timezone"), equalTo("America/New_York"));
-        assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549)));
-        assertThat(geoData.get("accuracy_radius"), equalTo(27));
-        assertThat(geoData.get("postal_code"), equalTo("12037"));
-        assertThat(geoData.get("city_confidence"), equalTo(11));
-        assertThat(geoData.get("asn"), equalTo(14671L));
-        assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications"));
-        assertThat(geoData.get("network"), equalTo("74.209.16.0/20"));
-        assertThat(geoData.get("hosting_provider"), equalTo(false));
-        assertThat(geoData.get("tor_exit_node"), equalTo(false));
-        assertThat(geoData.get("anonymous_vpn"), equalTo(false));
-        assertThat(geoData.get("anonymous"), equalTo(false));
-        assertThat(geoData.get("public_proxy"), equalTo(false));
-        assertThat(geoData.get("residential_proxy"), equalTo(false));
-        assertThat(geoData.get("domain"), equalTo("frpt.net"));
-        assertThat(geoData.get("isp"), equalTo("Fairpoint Communications"));
-        assertThat(geoData.get("isp_organization_name"), equalTo("Fairpoint Communications"));
-        assertThat(geoData.get("user_type"), equalTo("residential"));
-        assertThat(geoData.get("connection_type"), equalTo("Cable/DSL"));
-        assertThat(geoData.get("registered_country_in_european_union"), equalTo(false));
-        assertThat(geoData.get("registered_country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("registered_country_name"), equalTo("United States"));
-    }
-
-    public void testIsp() throws Exception {
-        String ip = "149.101.100.1";
-        GeoIpProcessor processor = new GeoIpProcessor(
-            GEOIP_TYPE,
-            randomAlphaOfLength(10),
-            null,
-            "source_field",
-            loader("GeoIP2-ISP-Test.mmdb"),
-            () -> true,
-            "target_field",
-            ipDataLookupAll(Database.Isp),
-            false,
-            false,
-            "filename"
-        );
-
-        Map<String, Object> document = new HashMap<>();
-        document.put("source_field", ip);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        processor.execute(ingestDocument);
-
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(8));
-        assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("asn"), equalTo(6167L));
-        assertThat(geoData.get("organization_name"), equalTo("CELLCO-PART"));
-        assertThat(geoData.get("network"), equalTo("149.101.100.0/28"));
-        assertThat(geoData.get("isp"), equalTo("Verizon Wireless"));
-        assertThat(geoData.get("isp_organization_name"), equalTo("Verizon Wireless"));
-        assertThat(geoData.get("mobile_network_code"), equalTo("004"));
-        assertThat(geoData.get("mobile_country_code"), equalTo("310"));
-    }
-
     public void testAddressIsNotInTheDatabase() throws Exception {
         GeoIpProcessor processor = new GeoIpProcessor(
             GEOIP_TYPE,
@@ -580,7 +206,7 @@ public void testAddressIsNotInTheDatabase() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "filename"
@@ -594,9 +220,9 @@ public void testAddressIsNotInTheDatabase() throws Exception {
     }
 
     /**
-     * Don't silently do DNS lookups or anything trappy on bogus data
+     * Tests that an exception in the IpDataLookup is propagated out of the GeoIpProcessor's execute method
      */
-    public void testInvalid() {
+    public void testExceptionPropagates() {
         GeoIpProcessor processor = new GeoIpProcessor(
             GEOIP_TYPE,
             randomAlphaOfLength(10),
@@ -605,7 +231,7 @@ public void testInvalid() {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "filename"
@@ -627,7 +253,7 @@ public void testListAllValid() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "filename"
@@ -639,11 +265,11 @@ public void testListAllValid() throws Exception {
         processor.execute(ingestDocument);
 
         @SuppressWarnings("unchecked")
-        List<Map<String, Object>> geoData = (List<Map<String, Object>>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(2));
-        assertThat(geoData.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
-        assertThat(geoData.get(1).get("city_name"), equalTo("Hoensbroek"));
+        List<Map<String, Object>> data = (List<Map<String, Object>>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(data, notNullValue());
+        assertThat(data.size(), equalTo(2));
+        assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
+        assertThat(data.get(1).get("city_name"), equalTo("Hoensbroek"));
     }
 
     public void testListPartiallyValid() throws Exception {
@@ -655,7 +281,7 @@ public void testListPartiallyValid() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "filename"
@@ -667,11 +293,11 @@ public void testListPartiallyValid() throws Exception {
         processor.execute(ingestDocument);
 
         @SuppressWarnings("unchecked")
-        List<Map<String, Object>> geoData = (List<Map<String, Object>>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(2));
-        assertThat(geoData.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
-        assertThat(geoData.get(1), nullValue());
+        List<Map<String, Object>> data = (List<Map<String, Object>>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(data, notNullValue());
+        assertThat(data.size(), equalTo(2));
+        assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
+        assertThat(data.get(1), nullValue());
     }
 
     public void testListNoMatches() throws Exception {
@@ -683,7 +309,7 @@ public void testListNoMatches() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "filename"
@@ -703,7 +329,7 @@ public void testListDatabaseReferenceCounting() throws Exception {
         GeoIpProcessor processor = new GeoIpProcessor(GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", () -> {
             loader.preLookup();
             return loader;
-        }, () -> true, "target_field", ipDataLookupAll(Database.City), false, false, "filename");
+        }, () -> true, "target_field", getMaxmindCityLookup(), false, false, "filename");
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", List.of("8.8.8.8", "82.171.64.0"));
@@ -711,11 +337,11 @@ public void testListDatabaseReferenceCounting() throws Exception {
         processor.execute(ingestDocument);
 
         @SuppressWarnings("unchecked")
-        List<Map<String, Object>> geoData = (List<Map<String, Object>>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(2));
-        assertThat(geoData.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
-        assertThat(geoData.get(1).get("city_name"), equalTo("Hoensbroek"));
+        List<Map<String, Object>> data = (List<Map<String, Object>>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(data, notNullValue());
+        assertThat(data.size(), equalTo(2));
+        assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
+        assertThat(data.get(1).get("city_name"), equalTo("Hoensbroek"));
 
         // Check the loader's reference count and attempt to close
         assertThat(loader.current(), equalTo(0));
@@ -732,7 +358,7 @@ public void testListFirstOnly() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             true,
             "filename"
@@ -744,9 +370,9 @@ public void testListFirstOnly() throws Exception {
         processor.execute(ingestDocument);
 
         @SuppressWarnings("unchecked")
-        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData, notNullValue());
-        assertThat(geoData.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
+        Map<String, Object> data = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(data, notNullValue());
+        assertThat(data.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
     }
 
     public void testListFirstOnlyNoMatches() throws Exception {
@@ -758,7 +384,7 @@ public void testListFirstOnlyNoMatches() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             true,
             "filename"
@@ -781,7 +407,7 @@ public void testInvalidDatabase() throws Exception {
             loader("GeoLite2-City.mmdb"),
             () -> false,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             true,
             "filename"
@@ -805,7 +431,7 @@ public void testNoDatabase() throws Exception {
             () -> null,
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             false,
             false,
             "GeoLite2-City"
@@ -829,7 +455,7 @@ public void testNoDatabase_ignoreMissing() throws Exception {
             () -> null,
             () -> true,
             "target_field",
-            ipDataLookupAll(Database.City),
+            getMaxmindCityLookup(),
             true,
             false,
             "GeoLite2-City"
@@ -843,13 +469,24 @@ public void testNoDatabase_ignoreMissing() throws Exception {
         assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
+    private static IpDataLookup getMaxmindCityLookup() {
+        final var database = Database.City;
+        return MaxmindIpDataLookups.getMaxmindLookup(database).apply(database.properties());
+    }
+
+    private static IpDataLookup getIpinfoGeolocationLookup() {
+        final var database = Database.CityV2;
+        return IpinfoIpDataLookups.getIpinfoLookup(database).apply(database.properties());
+    }
+
     private CheckedSupplier<IpDatabase, IOException> loader(final String path) {
         var loader = loader(path, null);
         return () -> loader;
     }
 
     private DatabaseReaderLazyLoader loader(final String databaseName, final AtomicBoolean closed) {
-        Path path = tmpDir.resolve(databaseName);
+        int last = databaseName.lastIndexOf("/");
+        final Path path = tmpDir.resolve(last == -1 ? databaseName : databaseName.substring(last + 1));
         copyDatabase(databaseName, path);
 
         final GeoIpCache cache = new GeoIpCache(1000);
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java
index 43ed96afb07e4..f4a3cfbde4f4c 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java
@@ -47,6 +47,7 @@ public static void startServer() throws Throwable {
         server.createContext("/404/", exchange -> {
             try {
                 exchange.sendResponseHeaders(404, 0);
+                exchange.close();
             } catch (Exception e) {
                 fail(e);
             }
@@ -102,6 +103,7 @@ public boolean checkCredentials(String username, String password) {
                     exchange.getResponseHeaders().add("Location", "/" + destination + "/");
                 }
                 exchange.sendResponseHeaders(302, 0);
+                exchange.close();
             } catch (Exception e) {
                 fail(e);
             }
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java
index 4ecf3056db738..d0cdc5a3e1b5e 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java
@@ -15,15 +15,9 @@
 
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.After;
 import org.junit.Before;
 
@@ -37,11 +31,12 @@
 
 import static java.util.Map.entry;
 import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
+import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.ipinfoTypeCleanup;
 import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn;
 import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseBoolean;
 import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseLocationDouble;
 import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -50,41 +45,19 @@
 
 public class IpinfoIpDataLookupsTests extends ESTestCase {
 
-    private ThreadPool threadPool;
-    private ResourceWatcherService resourceWatcherService;
-
     // a temporary directory that mmdb files can be copied to and read from
     private Path tmpDir;
 
     @Before
     public void setup() {
-        threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName());
-        Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build();
-        resourceWatcherService = new ResourceWatcherService(settings, threadPool);
         tmpDir = createTempDir();
     }
 
     @After
     public void cleanup() throws IOException {
-        resourceWatcherService.close();
-        threadPool.shutdownNow();
         IOUtils.rm(tmpDir);
     }
 
-    public void testDatabasePropertyInvariants() {
-        // the second ASN variant database is like a specialization of the ASN database
-        assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty()));
-        assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties()));
-
-        // the second City variant database is like a version of the ordinary City database but lacking many fields
-        assertThat(Sets.difference(Database.CityV2.properties(), Database.City.properties()), is(empty()));
-        assertThat(Sets.difference(Database.CityV2.defaultProperties(), Database.City.defaultProperties()), is(empty()));
-
-        // the second Country variant database is like a version of the ordinary Country database but lacking come fields
-        assertThat(Sets.difference(Database.CountryV2.properties(), Database.CountryV2.properties()), is(empty()));
-        assertThat(Database.CountryV2.defaultProperties(), equalTo(Database.Country.defaultProperties()));
-    }
-
     public void testParseAsn() {
         // expected case: "AS123" is 123
         assertThat(parseAsn("AS123"), equalTo(123L));
@@ -126,53 +99,42 @@ public void testParseLocationDouble() {
         assertThat(parseLocationDouble("anythingelse"), nullValue());
     }
 
-    public void testAsn() throws IOException {
+    public void testAsnFree() {
         assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
-        Path configDir = tmpDir;
-        copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb"));
-        copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb"));
-
-        GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
-        ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
-        configDatabases.initialize(resourceWatcherService);
-
-        // this is the 'free' ASN database (sample)
-        try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_asn_sample.mmdb")) {
-            IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Database.AsnV2.properties());
-            Map<String, Object> data = lookup.getData(loader, "5.182.109.0");
-            assertThat(
-                data,
-                equalTo(
-                    Map.ofEntries(
-                        entry("ip", "5.182.109.0"),
-                        entry("organization_name", "M247 Europe SRL"),
-                        entry("asn", 9009L),
-                        entry("network", "5.182.109.0/24"),
-                        entry("domain", "m247.com")
-                    )
-                )
-            );
-        }
+        String databaseName = "ip_asn_sample.mmdb";
+        String ip = "23.32.184.0";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("organization_name", "Akamai Technologies, Inc."),
+                entry("asn", 16625L),
+                entry("network", "23.32.184.0/21"),
+                entry("domain", "akamai.com")
+            )
+        );
+    }
 
-        // this is the non-free or 'standard' ASN database (sample)
-        try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("asn_sample.mmdb")) {
-            IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Database.AsnV2.properties());
-            Map<String, Object> data = lookup.getData(loader, "23.53.116.0");
-            assertThat(
-                data,
-                equalTo(
-                    Map.ofEntries(
-                        entry("ip", "23.53.116.0"),
-                        entry("organization_name", "Akamai Technologies, Inc."),
-                        entry("asn", 32787L),
-                        entry("network", "23.53.116.0/24"),
-                        entry("domain", "akamai.com"),
-                        entry("type", "hosting"),
-                        entry("country_iso_code", "US")
-                    )
-                )
-            );
-        }
+    public void testAsnStandard() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "asn_sample.mmdb";
+        String ip = "69.19.224.0";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("organization_name", "TPx Communications"),
+                entry("asn", 14265L),
+                entry("network", "69.19.224.0/22"),
+                entry("domain", "tpx.com"),
+                entry("type", "hosting"),
+                entry("country_iso_code", "US")
+            )
+        );
     }
 
     public void testAsnInvariants() {
@@ -212,149 +174,120 @@ public void testAsnInvariants() {
         }
     }
 
-    public void testCountry() throws IOException {
+    public void testCountryFree() {
         assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
-        Path configDir = tmpDir;
-        copyDatabase("ipinfo/ip_country_sample.mmdb", configDir.resolve("ip_country_sample.mmdb"));
-
-        GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
-        ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
-        configDatabases.initialize(resourceWatcherService);
-
-        // this is the 'free' Country database (sample)
-        try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb")) {
-            IpDataLookup lookup = new IpinfoIpDataLookups.Country(Database.CountryV2.properties());
-            Map<String, Object> data = lookup.getData(loader, "4.221.143.168");
-            assertThat(
-                data,
-                equalTo(
-                    Map.ofEntries(
-                        entry("ip", "4.221.143.168"),
-                        entry("country_name", "South Africa"),
-                        entry("country_iso_code", "ZA"),
-                        entry("continent_name", "Africa"),
-                        entry("continent_code", "AF")
-                    )
-                )
-            );
-        }
+        String databaseName = "ip_country_sample.mmdb";
+        String ip = "20.33.76.0";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new IpinfoIpDataLookups.Country(Database.CountryV2.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("country_name", "Ireland"),
+                entry("country_iso_code", "IE"),
+                entry("continent_name", "Europe"),
+                entry("continent_code", "EU")
+            )
+        );
     }
 
-    public void testGeolocation() throws IOException {
+    public void testGeolocationStandard() {
         assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
-        Path configDir = tmpDir;
-        copyDatabase("ipinfo/ip_geolocation_sample.mmdb", configDir.resolve("ip_geolocation_sample.mmdb"));
-
-        GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
-        ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
-        configDatabases.initialize(resourceWatcherService);
-
-        // this is the non-free or 'standard' Geolocation database (sample)
-        try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_geolocation_sample.mmdb")) {
-            IpDataLookup lookup = new IpinfoIpDataLookups.Geolocation(Database.CityV2.properties());
-            Map<String, Object> data = lookup.getData(loader, "2.124.90.182");
-            assertThat(
-                data,
-                equalTo(
-                    Map.ofEntries(
-                        entry("ip", "2.124.90.182"),
-                        entry("country_iso_code", "GB"),
-                        entry("region_name", "England"),
-                        entry("city_name", "London"),
-                        entry("timezone", "Europe/London"),
-                        entry("postal_code", "E1W"),
-                        entry("location", Map.of("lat", 51.50853, "lon", -0.12574))
-                    )
-                )
-            );
-        }
+        String databaseName = "ip_geolocation_standard_sample.mmdb";
+        String ip = "62.69.48.19";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new IpinfoIpDataLookups.Geolocation(Database.CityV2.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("country_iso_code", "GB"),
+                entry("region_name", "England"),
+                entry("city_name", "London"),
+                entry("timezone", "Europe/London"),
+                entry("postal_code", "E1W"),
+                entry("location", Map.of("lat", 51.50853, "lon", -0.12574))
+            )
+        );
     }
 
     public void testGeolocationInvariants() {
         assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
         Path configDir = tmpDir;
-        copyDatabase("ipinfo/ip_geolocation_sample.mmdb", configDir.resolve("ip_geolocation_sample.mmdb"));
+        copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", configDir.resolve("ip_geolocation_standard_sample.mmdb"));
 
         {
             final Set<String> expectedColumns = Set.of(
-                "network",
                 "city",
+                "geoname_id",
                 "region",
+                "region_code",
                 "country",
                 "postal_code",
                 "timezone",
-                "latitude",
-                "longitude"
+                "lat",
+                "lng"
             );
 
-            Path databasePath = configDir.resolve("ip_geolocation_sample.mmdb");
+            Path databasePath = configDir.resolve("ip_geolocation_standard_sample.mmdb");
             assertDatabaseInvariants(databasePath, (ip, row) -> {
                 assertThat(row.keySet(), equalTo(expectedColumns));
                 {
-                    String latitude = (String) row.get("latitude");
+                    String latitude = (String) row.get("lat");
                     assertThat(latitude, equalTo(latitude.trim()));
                     Double parsed = parseLocationDouble(latitude);
                     assertThat(parsed, notNullValue());
-                    assertThat(latitude, equalTo(Double.toString(parsed))); // reverse it
+                    assertThat(Double.parseDouble(latitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it
                 }
                 {
-                    String longitude = (String) row.get("longitude");
+                    String longitude = (String) row.get("lng");
                     assertThat(longitude, equalTo(longitude.trim()));
                     Double parsed = parseLocationDouble(longitude);
                     assertThat(parsed, notNullValue());
-                    assertThat(longitude, equalTo(Double.toString(parsed))); // reverse it
+                    assertThat(Double.parseDouble(longitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it
                 }
             });
         }
     }
 
-    public void testPrivacyDetection() throws IOException {
+    public void testPrivacyDetectionStandard() {
         assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
-        Path configDir = tmpDir;
-        copyDatabase("ipinfo/privacy_detection_sample.mmdb", configDir.resolve("privacy_detection_sample.mmdb"));
-
-        GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
-        ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
-        configDatabases.initialize(resourceWatcherService);
-
-        // testing the first row in the sample database
-        try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
-            IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
-            Map<String, Object> data = lookup.getData(loader, "1.53.59.33");
-            assertThat(
-                data,
-                equalTo(
-                    Map.ofEntries(
-                        entry("ip", "1.53.59.33"),
-                        entry("hosting", false),
-                        entry("proxy", false),
-                        entry("relay", false),
-                        entry("tor", false),
-                        entry("vpn", true)
-                    )
-                )
-            );
-        }
+        String databaseName = "privacy_detection_sample.mmdb";
+        String ip = "2.57.109.154";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("hosting", false),
+                entry("proxy", false),
+                entry("relay", false),
+                entry("tor", false),
+                entry("vpn", true)
+            )
+        );
+    }
 
-        // testing a row with a non-empty service in the sample database
-        try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
-            IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
-            Map<String, Object> data = lookup.getData(loader, "216.131.74.65");
-            assertThat(
-                data,
-                equalTo(
-                    Map.ofEntries(
-                        entry("ip", "216.131.74.65"),
-                        entry("hosting", true),
-                        entry("proxy", false),
-                        entry("service", "FastVPN"),
-                        entry("relay", false),
-                        entry("tor", false),
-                        entry("vpn", true)
-                    )
-                )
-            );
-        }
+    public void testPrivacyDetectionStandardNonEmptyService() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "privacy_detection_sample.mmdb";
+        String ip = "59.29.201.246";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("hosting", false),
+                entry("proxy", false),
+                entry("service", "VPNGate"),
+                entry("relay", false),
+                entry("tor", false),
+                entry("vpn", true)
+            )
+        );
     }
 
     public void testPrivacyDetectionInvariants() {
@@ -378,6 +311,107 @@ public void testPrivacyDetectionInvariants() {
         }
     }
 
+    public void testIpinfoTypeCleanup() {
+        Map<String, String> typesToCleanedTypes = Map.ofEntries(
+            // database_type strings from upstream:
+            // abuse.mmdb
+            entry("ipinfo standard_abuse_mmdb_v4.mmdb", "abuse_v4"),
+            // asn.mmdb
+            entry("ipinfo generic_asn_mmdb_v4.mmdb", "asn_v4"),
+            // carrier.mmdb
+            entry("ipinfo standard_carrier_mmdb.mmdb", "carrier"),
+            // location_extended_v2.mmdb
+            entry("ipinfo extended_location_v2.mmdb", "location_v2"),
+            // privacy_extended_v2.mmdb
+            entry("ipinfo extended_privacy_v2.mmdb", "privacy_v2"),
+            // standard_company.mmdb
+            entry("ipinfo standard_company.mmdb", "company"),
+            // standard_ip_hosted_domains_sample.mmdb
+            entry("ipinfo standard_ip_hosted_domains_sample.mmdb", "hosted_domains"),
+            // standard_location.mmdb
+            entry("ipinfo standard_location_mmdb_v4.mmdb", "location_v4"),
+            // standard_privacy.mmdb
+            entry("ipinfo standard_privacy.mmdb", "privacy"),
+
+            // database_type strings from test files:
+            // ip_asn_sample.mmdb
+            entry("ipinfo ip_asn_sample.mmdb", "asn"),
+            // ip_country_asn_sample.mmdb
+            entry("ipinfo ip_country_asn_sample.mmdb", "country_asn"),
+            // ip_geolocation_sample.mmdb
+            entry("ipinfo ip_geolocation_sample.mmdb", "geolocation"),
+            // abuse_contact_sample.mmdb
+            entry("ipinfo abuse_contact_sample.mmdb", "abuse_contact"),
+            // asn_sample.mmdb
+            entry("ipinfo asn_sample.mmdb", "asn"),
+            // hosted_domains_sample.mmdb
+            entry("ipinfo hosted_domains_sample.mmdb", "hosted_domains"),
+            // ip_carrier_sample.mmdb
+            entry("ipinfo ip_carrier_sample.mmdb", "carrier"),
+            // ip_company_sample.mmdb
+            entry("ipinfo ip_company_sample.mmdb", "company"),
+            // ip_country_sample.mmdb
+            entry("ipinfo ip_country_sample.mmdb", "country"),
+            // ip_geolocation_extended_ipv4_sample.mmdb
+            entry("ipinfo ip_geolocation_extended_ipv4_sample.mmdb", "geolocation_ipv4"),
+            // ip_geolocation_extended_ipv6_sample.mmdb
+            entry("ipinfo ip_geolocation_extended_ipv6_sample.mmdb", "geolocation_ipv6"),
+            // ip_geolocation_extended_sample.mmdb
+            entry("ipinfo ip_geolocation_extended_sample.mmdb", "geolocation"),
+            // ip_rdns_domains_sample.mmdb
+            entry("ipinfo ip_rdns_domains_sample.mmdb", "rdns_domains"),
+            // ip_rdns_hostnames_sample.mmdb
+            entry("ipinfo ip_rdns_hostnames_sample.mmdb", "rdns_hostnames"),
+            // privacy_detection_extended_sample.mmdb
+            entry("ipinfo privacy_detection_extended_sample.mmdb", "privacy_detection"),
+            // privacy_detection_sample.mmdb
+            entry("ipinfo privacy_detection_sample.mmdb", "privacy_detection"),
+
+            // database_type strings from downloaded (free) files:
+            // asn.mmdb
+            entry("ipinfo generic_asn_free.mmdb", "asn"),
+            // country.mmdb
+            entry("ipinfo generic_country_free.mmdb", "country"),
+            // country_asn.mmdb
+            entry("ipinfo generic_country_free_country_asn.mmdb", "country_country_asn")
+        );
+
+        for (var entry : typesToCleanedTypes.entrySet()) {
+            String type = entry.getKey();
+            String cleanedType = entry.getValue();
+            assertThat(ipinfoTypeCleanup(type), equalTo(cleanedType));
+        }
+    }
+
+    public void testDatabaseTypeParsing() throws IOException {
+        // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of
+        // database_type strings from these files, *and* it's also testing that we dispatch on those strings
+        // correctly and associated those files with the correct high-level Elasticsearch Database type.
+        // down the road it would probably make sense to split these out and find a better home for some of the
+        // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially
+        // pedantic about where precisely it should be.
+
+        copyDatabase("ipinfo/ip_asn_sample.mmdb", tmpDir.resolve("ip_asn_sample.mmdb"));
+        copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", tmpDir.resolve("ip_geolocation_standard_sample.mmdb"));
+        copyDatabase("ipinfo/asn_sample.mmdb", tmpDir.resolve("asn_sample.mmdb"));
+        copyDatabase("ipinfo/ip_country_sample.mmdb", tmpDir.resolve("ip_country_sample.mmdb"));
+        copyDatabase("ipinfo/privacy_detection_sample.mmdb", tmpDir.resolve("privacy_detection_sample.mmdb"));
+
+        assertThat(parseDatabaseFromType("ip_asn_sample.mmdb"), is(Database.AsnV2));
+        assertThat(parseDatabaseFromType("ip_geolocation_standard_sample.mmdb"), is(Database.CityV2));
+        assertThat(parseDatabaseFromType("asn_sample.mmdb"), is(Database.AsnV2));
+        assertThat(parseDatabaseFromType("ip_country_sample.mmdb"), is(Database.CountryV2));
+        assertThat(parseDatabaseFromType("privacy_detection_sample.mmdb"), is(Database.PrivacyDetection));
+
+        // additional cases where we're bailing early on types we don't support
+        assertThat(IpDataLookupFactories.getDatabase("ipinfo ip_country_asn_sample.mmdb"), nullValue());
+        assertThat(IpDataLookupFactories.getDatabase("ipinfo privacy_detection_extended_sample.mmdb"), nullValue());
+    }
+
+    private Database parseDatabaseFromType(String databaseFile) throws IOException {
+        return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile)));
+    }
+
     private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer<InetAddress, Map<String, Object>> rowConsumer) {
         try (Reader reader = new Reader(pathToFile(databasePath))) {
             Networks<?> networks = reader.networks(Map.class);
@@ -403,4 +437,29 @@ private static void assertDatabaseInvariants(final Path databasePath, final BiCo
     private static File pathToFile(Path databasePath) {
         return databasePath.toFile();
     }
+
+    private void assertExpectedLookupResults(String databaseName, String ip, IpDataLookup lookup, Map<String, Object> expected) {
+        try (DatabaseReaderLazyLoader loader = loader(databaseName)) {
+            Map<String, Object> actual = lookup.getData(loader, ip);
+            assertThat(
+                "The set of keys in the result are not the same as the set of expected keys",
+                actual.keySet(),
+                containsInAnyOrder(expected.keySet().toArray(new String[0]))
+            );
+            for (Map.Entry<String, Object> entry : expected.entrySet()) {
+                assertThat("Unexpected value for key [" + entry.getKey() + "]", actual.get(entry.getKey()), equalTo(entry.getValue()));
+            }
+        } catch (AssertionError e) {
+            fail(e, "Assert failed for database [%s] with address [%s]", databaseName, ip);
+        } catch (Exception e) {
+            fail(e, "Exception for database [%s] with address [%s]", databaseName, ip);
+        }
+    }
+
+    private DatabaseReaderLazyLoader loader(final String databaseName) {
+        Path path = tmpDir.resolve(databaseName);
+        copyDatabase("ipinfo/" + databaseName, path); // the ipinfo databases are prefixed on the test classpath
+        final GeoIpCache cache = new GeoIpCache(1000);
+        return new DatabaseReaderLazyLoader(cache, path, null);
+    }
 }
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java
index 46a34c2cdad56..083e9b5bc32da 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java
@@ -83,39 +83,4 @@ public void testIsGzip() throws IOException {
         assertThat(MMDBUtil.isGzip(database), is(false));
         assertThat(MMDBUtil.isGzip(gzipDatabase), is(true));
     }
-
-    public void testDatabaseTypeParsing() throws IOException {
-        // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of
-        // database_type strings from these files, *and* it's also testing that we dispatch on those strings
-        // correctly and associated those files with the correct high-level Elasticsearch Database type.
-        // down the road it would probably make sense to split these out and find a better home for some of the
-        // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially
-        // pedantic about where precisely it should be.
-
-        copyDatabase("GeoLite2-City-Test.mmdb", tmpDir);
-        copyDatabase("GeoLite2-Country-Test.mmdb", tmpDir);
-        copyDatabase("GeoLite2-ASN-Test.mmdb", tmpDir);
-        copyDatabase("GeoIP2-Anonymous-IP-Test.mmdb", tmpDir);
-        copyDatabase("GeoIP2-City-Test.mmdb", tmpDir);
-        copyDatabase("GeoIP2-Country-Test.mmdb", tmpDir);
-        copyDatabase("GeoIP2-Connection-Type-Test.mmdb", tmpDir);
-        copyDatabase("GeoIP2-Domain-Test.mmdb", tmpDir);
-        copyDatabase("GeoIP2-Enterprise-Test.mmdb", tmpDir);
-        copyDatabase("GeoIP2-ISP-Test.mmdb", tmpDir);
-
-        assertThat(parseDatabaseFromType("GeoLite2-City-Test.mmdb"), is(Database.City));
-        assertThat(parseDatabaseFromType("GeoLite2-Country-Test.mmdb"), is(Database.Country));
-        assertThat(parseDatabaseFromType("GeoLite2-ASN-Test.mmdb"), is(Database.Asn));
-        assertThat(parseDatabaseFromType("GeoIP2-Anonymous-IP-Test.mmdb"), is(Database.AnonymousIp));
-        assertThat(parseDatabaseFromType("GeoIP2-City-Test.mmdb"), is(Database.City));
-        assertThat(parseDatabaseFromType("GeoIP2-Country-Test.mmdb"), is(Database.Country));
-        assertThat(parseDatabaseFromType("GeoIP2-Connection-Type-Test.mmdb"), is(Database.ConnectionType));
-        assertThat(parseDatabaseFromType("GeoIP2-Domain-Test.mmdb"), is(Database.Domain));
-        assertThat(parseDatabaseFromType("GeoIP2-Enterprise-Test.mmdb"), is(Database.Enterprise));
-        assertThat(parseDatabaseFromType("GeoIP2-ISP-Test.mmdb"), is(Database.Isp));
-    }
-
-    private Database parseDatabaseFromType(String databaseFile) throws IOException {
-        return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile)));
-    }
 }
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java
new file mode 100644
index 0000000000000..57ee2191a590d
--- /dev/null
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java
@@ -0,0 +1,339 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.ingest.geoip;
+
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Map;
+
+import static java.util.Map.entry;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class MaxmindIpDataLookupsTests extends ESTestCase {
+
+    // a temporary directory that mmdb files can be copied to and read from
+    private Path tmpDir;
+
+    @Before
+    public void setup() {
+        tmpDir = createTempDir();
+    }
+
+    @After
+    public void cleanup() throws IOException {
+        IOUtils.rm(tmpDir);
+    }
+
+    public void testCity() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoLite2-City.mmdb";
+        String ip = "8.8.8.8";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.City(Database.City.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("country_in_european_union", false),
+                entry("country_iso_code", "US"),
+                entry("country_name", "United States"),
+                entry("continent_code", "NA"),
+                entry("continent_name", "North America"),
+                entry("timezone", "America/Chicago"),
+                entry("location", Map.of("lat", 37.751d, "lon", -97.822d)),
+                entry("accuracy_radius", 1000),
+                entry("registered_country_in_european_union", false),
+                entry("registered_country_iso_code", "US"),
+                entry("registered_country_name", "United States")
+            )
+        );
+    }
+
+    public void testCity_withIpV6() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoLite2-City.mmdb";
+        String ip = "2602:306:33d3:8000::3257:9652";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.City(Database.City.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("country_in_european_union", false),
+                entry("country_iso_code", "US"),
+                entry("country_name", "United States"),
+                entry("continent_code", "NA"),
+                entry("continent_name", "North America"),
+                entry("region_iso_code", "US-FL"),
+                entry("region_name", "Florida"),
+                entry("city_name", "Homestead"),
+                entry("postal_code", "33035"),
+                entry("timezone", "America/New_York"),
+                entry("location", Map.of("lat", 25.4573d, "lon", -80.4572d)),
+                entry("accuracy_radius", 50),
+                entry("registered_country_in_european_union", false),
+                entry("registered_country_iso_code", "US"),
+                entry("registered_country_name", "United States")
+            )
+        );
+    }
+
+    public void testCityWithMissingLocation() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoLite2-City.mmdb";
+        String ip = "80.231.5.0";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.City(Database.City.properties()),
+            Map.ofEntries(entry("ip", ip))
+        );
+    }
+
+    public void testCountry() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoLite2-Country.mmdb";
+        String ip = "82.170.213.79";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.Country(Database.Country.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("country_in_european_union", true),
+                entry("country_iso_code", "NL"),
+                entry("country_name", "Netherlands"),
+                entry("continent_code", "EU"),
+                entry("continent_name", "Europe"),
+                entry("registered_country_in_european_union", true),
+                entry("registered_country_iso_code", "NL"),
+                entry("registered_country_name", "Netherlands")
+            )
+        );
+    }
+
+    /**
+     * Don't silently do DNS lookups or anything trappy on bogus data
+     */
+    public void testInvalid() throws IOException {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoLite2-Country.mmdb";
+        String ip = "www.google.com";
+        try (DatabaseReaderLazyLoader loader = loader(databaseName)) {
+            IpDataLookup lookup = new MaxmindIpDataLookups.Country(Database.Country.properties());
+            IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> lookup.getData(loader, ip));
+            assertThat(e.getMessage(), containsString("not an IP string literal"));
+        }
+    }
+
+    public void testCountryWithMissingLocation() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoLite2-Country.mmdb";
+        String ip = "80.231.5.0";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.Country(Database.Country.properties()),
+            Map.ofEntries(entry("ip", ip))
+        );
+    }
+
+    public void testAsn() throws IOException {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoLite2-ASN.mmdb";
+        String ip = "82.171.64.0";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.Asn(Database.Asn.properties()),
+            Map.ofEntries(entry("ip", ip), entry("organization_name", "KPN B.V."), entry("asn", 1136L), entry("network", "82.168.0.0/14"))
+        );
+    }
+
+    public void testAnonymousIp() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoIP2-Anonymous-IP-Test.mmdb";
+        String ip = "81.2.69.1";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.AnonymousIp(Database.AnonymousIp.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("hosting_provider", true),
+                entry("tor_exit_node", true),
+                entry("anonymous_vpn", true),
+                entry("anonymous", true),
+                entry("public_proxy", true),
+                entry("residential_proxy", true)
+            )
+        );
+    }
+
+    public void testConnectionType() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoIP2-Connection-Type-Test.mmdb";
+        String ip = "214.78.120.5";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.ConnectionType(Database.ConnectionType.properties()),
+            Map.ofEntries(entry("ip", ip), entry("connection_type", "Satellite"))
+        );
+    }
+
+    public void testDomain() {
+        String databaseName = "GeoIP2-Domain-Test.mmdb";
+        String ip = "69.219.64.2";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.Domain(Database.Domain.properties()),
+            Map.ofEntries(entry("ip", ip), entry("domain", "ameritech.net"))
+        );
+    }
+
+    public void testEnterprise() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoIP2-Enterprise-Test.mmdb";
+        String ip = "74.209.24.4";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.Enterprise(Database.Enterprise.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("country_confidence", 99),
+                entry("country_in_european_union", false),
+                entry("country_iso_code", "US"),
+                entry("country_name", "United States"),
+                entry("continent_code", "NA"),
+                entry("continent_name", "North America"),
+                entry("region_iso_code", "US-NY"),
+                entry("region_name", "New York"),
+                entry("city_confidence", 11),
+                entry("city_name", "Chatham"),
+                entry("timezone", "America/New_York"),
+                entry("location", Map.of("lat", 42.3478, "lon", -73.5549)),
+                entry("accuracy_radius", 27),
+                entry("postal_code", "12037"),
+                entry("postal_confidence", 11),
+                entry("asn", 14671L),
+                entry("organization_name", "FairPoint Communications"),
+                entry("network", "74.209.16.0/20"),
+                entry("hosting_provider", false),
+                entry("tor_exit_node", false),
+                entry("anonymous_vpn", false),
+                entry("anonymous", false),
+                entry("public_proxy", false),
+                entry("residential_proxy", false),
+                entry("domain", "frpt.net"),
+                entry("isp", "Fairpoint Communications"),
+                entry("isp_organization_name", "Fairpoint Communications"),
+                entry("user_type", "residential"),
+                entry("connection_type", "Cable/DSL"),
+                entry("registered_country_in_european_union", false),
+                entry("registered_country_iso_code", "US"),
+                entry("registered_country_name", "United States")
+            )
+        );
+    }
+
+    public void testIsp() {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
+        String databaseName = "GeoIP2-ISP-Test.mmdb";
+        String ip = "149.101.100.1";
+        assertExpectedLookupResults(
+            databaseName,
+            ip,
+            new MaxmindIpDataLookups.Isp(Database.Isp.properties()),
+            Map.ofEntries(
+                entry("ip", ip),
+                entry("asn", 6167L),
+                entry("organization_name", "CELLCO-PART"),
+                entry("network", "149.101.100.0/28"),
+                entry("isp", "Verizon Wireless"),
+                entry("isp_organization_name", "Verizon Wireless"),
+                entry("mobile_network_code", "004"),
+                entry("mobile_country_code", "310")
+            )
+        );
+    }
+
+    public void testDatabaseTypeParsing() throws IOException {
+        // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of
+        // database_type strings from these files, *and* it's also testing that we dispatch on those strings
+        // correctly and associated those files with the correct high-level Elasticsearch Database type.
+        // down the road it would probably make sense to split these out and find a better home for some of the
+        // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially
+        // pedantic about where precisely it should be.
+
+        copyDatabase("GeoLite2-City-Test.mmdb", tmpDir);
+        copyDatabase("GeoLite2-Country-Test.mmdb", tmpDir);
+        copyDatabase("GeoLite2-ASN-Test.mmdb", tmpDir);
+        copyDatabase("GeoIP2-Anonymous-IP-Test.mmdb", tmpDir);
+        copyDatabase("GeoIP2-City-Test.mmdb", tmpDir);
+        copyDatabase("GeoIP2-Country-Test.mmdb", tmpDir);
+        copyDatabase("GeoIP2-Connection-Type-Test.mmdb", tmpDir);
+        copyDatabase("GeoIP2-Domain-Test.mmdb", tmpDir);
+        copyDatabase("GeoIP2-Enterprise-Test.mmdb", tmpDir);
+        copyDatabase("GeoIP2-ISP-Test.mmdb", tmpDir);
+
+        assertThat(parseDatabaseFromType("GeoLite2-City-Test.mmdb"), is(Database.City));
+        assertThat(parseDatabaseFromType("GeoLite2-Country-Test.mmdb"), is(Database.Country));
+        assertThat(parseDatabaseFromType("GeoLite2-ASN-Test.mmdb"), is(Database.Asn));
+        assertThat(parseDatabaseFromType("GeoIP2-Anonymous-IP-Test.mmdb"), is(Database.AnonymousIp));
+        assertThat(parseDatabaseFromType("GeoIP2-City-Test.mmdb"), is(Database.City));
+        assertThat(parseDatabaseFromType("GeoIP2-Country-Test.mmdb"), is(Database.Country));
+        assertThat(parseDatabaseFromType("GeoIP2-Connection-Type-Test.mmdb"), is(Database.ConnectionType));
+        assertThat(parseDatabaseFromType("GeoIP2-Domain-Test.mmdb"), is(Database.Domain));
+        assertThat(parseDatabaseFromType("GeoIP2-Enterprise-Test.mmdb"), is(Database.Enterprise));
+        assertThat(parseDatabaseFromType("GeoIP2-ISP-Test.mmdb"), is(Database.Isp));
+    }
+
+    private Database parseDatabaseFromType(String databaseFile) throws IOException {
+        return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile)));
+    }
+
+    private void assertExpectedLookupResults(String databaseName, String ip, IpDataLookup lookup, Map<String, Object> expected) {
+        try (DatabaseReaderLazyLoader loader = loader(databaseName)) {
+            Map<String, Object> actual = lookup.getData(loader, ip);
+            assertThat(
+                "The set of keys in the result are not the same as the set of expected keys",
+                actual.keySet(),
+                containsInAnyOrder(expected.keySet().toArray(new String[0]))
+            );
+            for (Map.Entry<String, Object> entry : expected.entrySet()) {
+                assertThat("Unexpected value for key [" + entry.getKey() + "]", actual.get(entry.getKey()), equalTo(entry.getValue()));
+            }
+        } catch (AssertionError e) {
+            fail(e, "Assert failed for database [%s] with address [%s]", databaseName, ip);
+        } catch (Exception e) {
+            fail(e, "Exception for database [%s] with address [%s]", databaseName, ip);
+        }
+    }
+
+    private DatabaseReaderLazyLoader loader(final String databaseName) {
+        Path path = tmpDir.resolve(databaseName);
+        copyDatabase(databaseName, path);
+        final GeoIpCache cache = new GeoIpCache(1000);
+        return new DatabaseReaderLazyLoader(cache, path, null);
+    }
+}
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java
index 33356ad4235dc..76b2896afe7a0 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin;
+import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Ipinfo;
 import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Local;
 import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Maxmind;
 import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Web;
@@ -21,6 +22,7 @@
 import java.io.IOException;
 import java.util.Set;
 
+import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.IPINFO_NAMES;
 import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.MAXMIND_NAMES;
 
 public class DatabaseConfigurationTests extends AbstractXContentSerializingTestCase<DatabaseConfiguration> {
@@ -44,13 +46,14 @@ protected DatabaseConfiguration createTestInstance() {
     }
 
     public static DatabaseConfiguration randomDatabaseConfiguration(String id) {
+        boolean useIpinfo = randomBoolean();
         DatabaseConfiguration.Provider provider = switch (between(0, 2)) {
-            case 0 -> new Maxmind(randomAlphaOfLength(5));
+            case 0 -> useIpinfo ? new Ipinfo() : new Maxmind(randomAlphaOfLength(5));
             case 1 -> new Web();
             case 2 -> new Local(randomAlphaOfLength(10));
             default -> throw new AssertionError("failure, got illegal switch case");
         };
-        return new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), provider);
+        return new DatabaseConfiguration(id, useIpinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES), provider);
     }
 
     @Override
@@ -61,21 +64,21 @@ protected DatabaseConfiguration mutateInstance(DatabaseConfiguration instance) {
             case 1:
                 return new DatabaseConfiguration(
                     instance.id(),
-                    randomValueOtherThan(instance.name(), () -> randomFrom(MAXMIND_NAMES)),
+                    randomValueOtherThan(
+                        instance.name(),
+                        () -> instance.provider() instanceof Ipinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES)
+                    ),
                     instance.provider()
                 );
             case 2:
                 DatabaseConfiguration.Provider provider = instance.provider();
-                DatabaseConfiguration.Provider modifiedProvider;
-                if (provider instanceof Maxmind maxmind) {
-                    modifiedProvider = new Maxmind(((Maxmind) instance.provider()).accountId() + randomAlphaOfLength(2));
-                } else if (provider instanceof Web) {
-                    modifiedProvider = new Maxmind(randomAlphaOfLength(20)); // can't modify a Web
-                } else if (provider instanceof Local local) {
-                    modifiedProvider = new Local(local.type() + randomAlphaOfLength(2));
-                } else {
-                    throw new AssertionError("Unexpected provider type: " + provider.getClass());
-                }
+                DatabaseConfiguration.Provider modifiedProvider = switch (provider) {
+                    case Maxmind maxmind -> new Maxmind(maxmind.accountId() + randomAlphaOfLength(2));
+                    case Ipinfo ignored -> new Local(randomAlphaOfLength(20)); // can't modify Ipinfo
+                    case Web ignored -> new Local(randomAlphaOfLength(20)); // can't modify a Web
+                    case Local local -> new Local(local.type() + randomAlphaOfLength(2));
+                    default -> throw new AssertionError("Unexpected provider type: " + provider.getClass());
+                };
                 return new DatabaseConfiguration(instance.id(), instance.name(), modifiedProvider);
             default:
                 throw new AssertionError("failure, got illegal switch case");
diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb
index 916a8252a5df1..289318a124d75 100644
Binary files a/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb and b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb
index 3e1fc49ba48a5..d2bac8452a0f2 100644
Binary files a/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb
index 88428315ee8d6..caa218f02770b 100644
Binary files a/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb
deleted file mode 100644
index ed738bdde1450..0000000000000
Binary files a/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb and /dev/null differ
diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb
new file mode 100644
index 0000000000000..205bd77fd53e2
Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb
index ac669536ae183..4f2fca5559e14 100644
Binary files a/modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb and b/modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb differ
diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml
index 04fd2ac6a8189..a1104505bc240 100644
--- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml
+++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml
@@ -1,8 +1,20 @@
+---
 setup:
   - requires:
       cluster_features: ["geoip.downloader.database.configuration", "get_database_configuration_action.multi_node"]
       reason: "geoip downloader database configuration APIs added in 8.15, and updated in 8.16 to return more results"
 
+---
+teardown:
+  - do:
+      ingest.delete_ip_location_database:
+        id: "my_database_1"
+        ignore: 404
+  - do:
+      ingest.delete_ip_location_database:
+        id: "my_database_2"
+        ignore: 404
+
 ---
 "Test adding, getting, and removing geoip databases":
   - do:
diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml
new file mode 100644
index 0000000000000..fd73c715a5ac5
--- /dev/null
+++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml
@@ -0,0 +1,45 @@
+setup:
+  - requires:
+      cluster_features:
+        - "put_database_configuration_action.ipinfo"
+      reason: "ipinfo support added in 8.16"
+
+---
+"Test ip_location processor with defaults":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "ip_location" : {
+                  "field" : "field1"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: {field1: "89.160.20.128"}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.field1: "89.160.20.128" }
+  - length: { _source.ip_location: 7 }
+  - match: { _source.ip_location.city_name: "Linköping" }
+  - match: { _source.ip_location.country_iso_code: "SE" }
+  - match: { _source.ip_location.location.lon: 15.6167 }
+  - match: { _source.ip_location.location.lat: 58.4167 }
+  - match: { _source.ip_location.region_iso_code: "SE-E" }
+  - match: { _source.ip_location.country_name: "Sweden" }
+  - match: { _source.ip_location.region_name: "Östergötland County" }
+  - match: { _source.ip_location.continent_name: "Europe" }
diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml
new file mode 100644
index 0000000000000..e2e9a1fdb5e28
--- /dev/null
+++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml
@@ -0,0 +1,137 @@
+---
+setup:
+  - requires:
+      cluster_features:
+        - "put_database_configuration_action.ipinfo"
+      reason: "ip location downloader database configuration APIs added in 8.16 to support more types"
+
+---
+teardown:
+  - do:
+      ingest.delete_ip_location_database:
+        id: "my_database_1"
+        ignore: 404
+  - do:
+      ingest.delete_ip_location_database:
+        id: "my_database_2"
+        ignore: 404
+  - do:
+      ingest.delete_ip_location_database:
+        id: "my_database_3"
+        ignore: 404
+
+---
+"Test adding, getting, and removing ip location databases":
+  - do:
+      ingest.put_ip_location_database:
+        id: "my_database_1"
+        body:  >
+          {
+            "name": "GeoIP2-City",
+            "maxmind": {
+              "account_id": "1234"
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_ip_location_database:
+        id: "my_database_1"
+        body:  >
+          {
+            "name": "GeoIP2-Country",
+            "maxmind": {
+              "account_id": "4321"
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_ip_location_database:
+        id: "my_database_2"
+        body:  >
+          {
+            "name": "GeoIP2-City",
+            "maxmind": {
+              "account_id": "1234"
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      catch: /illegal_argument_exception/
+      ingest.put_ip_location_database:
+        id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI="
+        body:  >
+          {
+            "name": "GeoIP2-City",
+            "web": {
+            }
+          }
+
+  - do:
+      ingest.put_ip_location_database:
+        id: "my_database_3"
+        body:  >
+          {
+            "name": "standard_privacy",
+            "ipinfo": {
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.get_ip_location_database:
+        id: "my_database_1"
+  - length: { databases: 1 }
+  - match: { databases.0.id: "my_database_1" }
+  - gte: { databases.0.modified_date_millis: 0 }
+  - match: { databases.0.database.name: "GeoIP2-Country" }
+  - match: { databases.0.database.maxmind.account_id: "4321" }
+
+  - do:
+      ingest.get_ip_location_database: {}
+  - length: { databases: 7 }
+
+  - do:
+      ingest.get_ip_location_database:
+        id: "my_database_1,my_database_2"
+  - length: { databases: 2 }
+
+  - do:
+      ingest.get_ip_location_database:
+        id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI="
+  - length: { databases: 1 }
+  - match: { databases.0.id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" }
+  - gte: { databases.0.modified_date_millis: -1 }
+  - match: { databases.0.database.name: "MyCustomGeoLite2-City" }
+
+  - do:
+      ingest.delete_ip_location_database:
+        id: "my_database_1"
+
+  - do:
+      catch: /resource_not_found_exception/
+      ingest.delete_ip_location_database:
+        id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI="
+
+  - do:
+      ingest.get_ip_location_database: {}
+  - length: { databases: 6 }
+
+  - do:
+      ingest.get_ip_location_database:
+        id: "my_database_2"
+  - length: { databases: 1 }
+  - match: { databases.0.id: "my_database_2" }
+  - gte: { databases.0.modified_date_millis: 0 }
+  - match: { databases.0.database.name: "GeoIP2-City" }
+  - match: { databases.0.database.maxmind.account_id: "1234" }
+
+  - do:
+      ingest.get_ip_location_database:
+        id: "my_database_3"
+  - length: { databases: 1 }
+  - match: { databases.0.id: "my_database_3" }
+  - gte: { databases.0.modified_date_millis: 0 }
+  - match: { databases.0.database.name: "standard_privacy" }
diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
index 61bd31fea3455..553e4696af316 100644
--- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
+++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
@@ -12,6 +12,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.settings.Settings;
@@ -37,6 +39,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.startsWith;
 
 /**
@@ -150,15 +153,15 @@ private void assertThreadPoolsBlocked() {
         new Thread(() -> expectThrows(EsRejectedExecutionException.class, () -> getFuture.actionGet(SAFE_AWAIT_TIMEOUT))).start();
 
         // intentionally commented out this test until https://github.com/elastic/elasticsearch/issues/97916 is fixed
-        // var e3 = expectThrows(
-        // SearchPhaseExecutionException.class,
-        // () -> client().prepareSearch(USER_INDEX)
-        // .setQuery(QueryBuilders.matchAllQuery())
-        // // Request times out if max concurrent shard requests is set to 1
-        // .setMaxConcurrentShardRequests(usually() ? SearchRequest.DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS : randomIntBetween(2, 10))
-        // .get()
-        // );
-        // assertThat(e3.getMessage(), containsString("all shards failed"));
+        var e3 = expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> client().prepareSearch(USER_INDEX)
+                .setQuery(QueryBuilders.matchAllQuery())
+                // Request times out if max concurrent shard requests is set to 1
+                .setMaxConcurrentShardRequests(usually() ? SearchRequest.DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS : randomIntBetween(2, 10))
+                .get()
+        );
+        assertThat(e3.getMessage(), containsString("all shards failed"));
     }
 
     protected void runWithBlockedThreadPools(Runnable runnable) throws Exception {
diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java
index 570c2a5f3783a..df6780aba7222 100644
--- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java
+++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java
@@ -81,7 +81,7 @@ public void testBasic() throws Exception {
         ensureGreen("test");
         prepareIndex("test").setId("1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get();
         assertResponse(buildRequest("doc['foo'] + 1"), rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D);
         });
     }
@@ -91,7 +91,7 @@ public void testFunction() throws Exception {
         ensureGreen("test");
         prepareIndex("test").setId("1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get();
         assertNoFailuresAndResponse(buildRequest("doc['foo'] + abs(1)"), rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D);
         });
     }
@@ -102,7 +102,7 @@ public void testBasicUsingDotValue() throws Exception {
 
         prepareIndex("test").setId("1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get();
         assertResponse(buildRequest("doc['foo'].value + 1"), rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D);
         });
     }
@@ -125,7 +125,7 @@ public void testScore() throws Exception {
         assertResponse(req, rsp -> {
             assertNoFailures(rsp);
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals("1", hits.getAt(0).getId());
             assertEquals("3", hits.getAt(1).getId());
             assertEquals("2", hits.getAt(2).getId());
@@ -148,25 +148,25 @@ public void testDateMethods() throws Exception {
             prepareIndex("test").setId("2").setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")
         );
         assertResponse(buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(-11.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         });
         assertResponse(buildRequest("doc['date0'].getHourOfDay() + doc['date1'].getDayOfMonth()"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(24.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         });
         assertResponse(buildRequest("doc['date1'].getMonth() + 1"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(9.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(10.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         });
         assertResponse(buildRequest("doc['date1'].getYear()"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(1985.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(1983.0, hits.getAt(1).field("foo").getValue(), 0.0D);
@@ -182,25 +182,25 @@ public void testDateObjectMethods() throws Exception {
             prepareIndex("test").setId("2").setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")
         );
         assertResponse(buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(-11.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         });
         assertResponse(buildRequest("doc['date0'].date.getHourOfDay() + doc['date1'].date.dayOfMonth"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(24.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         });
         assertResponse(buildRequest("doc['date1'].date.monthOfYear + 1"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(10.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(11.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         });
         assertResponse(buildRequest("doc['date1'].date.year"), rsp -> {
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             SearchHits hits = rsp.getHits();
             assertEquals(1985.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(1983.0, hits.getAt(1).field("foo").getValue(), 0.0D);
@@ -238,7 +238,7 @@ public void testMultiValueMethods() throws Exception {
 
         assertNoFailuresAndResponse(buildRequest("doc['double0'].count() + doc['double1'].count()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(2.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -246,7 +246,7 @@ public void testMultiValueMethods() throws Exception {
 
         assertNoFailuresAndResponse(buildRequest("doc['double0'].sum()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(7.5, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(6.0, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -254,7 +254,7 @@ public void testMultiValueMethods() throws Exception {
 
         assertNoFailuresAndResponse(buildRequest("doc['double0'].avg() + doc['double1'].avg()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(4.3, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(8.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(5.5, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -262,7 +262,7 @@ public void testMultiValueMethods() throws Exception {
 
         assertNoFailuresAndResponse(buildRequest("doc['double0'].median()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(1.5, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(1.25, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -270,7 +270,7 @@ public void testMultiValueMethods() throws Exception {
 
         assertNoFailuresAndResponse(buildRequest("doc['double0'].min()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(1.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(-1.5, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -278,7 +278,7 @@ public void testMultiValueMethods() throws Exception {
 
         assertNoFailuresAndResponse(buildRequest("doc['double0'].max()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -286,7 +286,7 @@ public void testMultiValueMethods() throws Exception {
 
         assertNoFailuresAndResponse(buildRequest("doc['double0'].sum()/doc['double0'].count()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(2.5, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(1.5, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -295,7 +295,7 @@ public void testMultiValueMethods() throws Exception {
         // make sure count() works for missing
         assertNoFailuresAndResponse(buildRequest("doc['double2'].count()"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(1.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(0.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(0.0, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -304,7 +304,7 @@ public void testMultiValueMethods() throws Exception {
         // make sure .empty works in the same way
         assertNoFailuresAndResponse(buildRequest("doc['double2'].empty ? 5.0 : 2.0"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(2.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(5.0, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -342,7 +342,7 @@ public void testSparseField() throws Exception {
         );
         assertNoFailuresAndResponse(buildRequest("doc['x'] + 1"), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(2, rsp.getHits().getTotalHits().value);
+            assertEquals(2, rsp.getHits().getTotalHits().value());
             assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(1.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         });
@@ -378,7 +378,7 @@ public void testParams() throws Exception {
         String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
         assertResponse(buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L), rsp -> {
             SearchHits hits = rsp.getHits();
-            assertEquals(3, hits.getTotalHits().value);
+            assertEquals(3, hits.getTotalHits().value());
             assertEquals(24.5, hits.getAt(0).field("foo").getValue(), 0.0D);
             assertEquals(9.5, hits.getAt(1).field("foo").getValue(), 0.0D);
             assertEquals(13.5, hits.getAt(2).field("foo").getValue(), 0.0D);
@@ -501,7 +501,7 @@ public void testSpecialValueVariable() throws Exception {
             );
 
         assertResponse(req, rsp -> {
-            assertEquals(3, rsp.getHits().getTotalHits().value);
+            assertEquals(3, rsp.getHits().getTotalHits().value());
 
             Stats stats = rsp.getAggregations().get("int_agg");
             assertEquals(39.0, stats.getMax(), 0.0001);
@@ -655,22 +655,22 @@ public void testGeo() throws Exception {
         refresh();
         // access .lat
         assertNoFailuresAndResponse(buildRequest("doc['location'].lat"), rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(61.5240, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
         });
         // access .lon
         assertNoFailuresAndResponse(buildRequest("doc['location'].lon"), rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(105.3188, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
         });
         // access .empty
         assertNoFailuresAndResponse(buildRequest("doc['location'].empty ? 1 : 0"), rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(0, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
         });
         // call haversin
         assertNoFailuresAndResponse(buildRequest("haversin(38.9072, 77.0369, doc['location'].lat, doc['location'].lon)"), rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(3170D, rsp.getHits().getAt(0).field("foo").getValue(), 50D);
         });
     }
@@ -693,14 +693,14 @@ public void testBoolean() throws Exception {
         );
         // access .value
         assertNoFailuresAndResponse(buildRequest("doc['vip'].value"), rsp -> {
-            assertEquals(3, rsp.getHits().getTotalHits().value);
+            assertEquals(3, rsp.getHits().getTotalHits().value());
             assertEquals(1.0D, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
             assertEquals(0.0D, rsp.getHits().getAt(1).field("foo").getValue(), 1.0D);
             assertEquals(0.0D, rsp.getHits().getAt(2).field("foo").getValue(), 1.0D);
         });
         // access .empty
         assertNoFailuresAndResponse(buildRequest("doc['vip'].empty ? 1 : 0"), rsp -> {
-            assertEquals(3, rsp.getHits().getTotalHits().value);
+            assertEquals(3, rsp.getHits().getTotalHits().value());
             assertEquals(0.0D, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
             assertEquals(0.0D, rsp.getHits().getAt(1).field("foo").getValue(), 1.0D);
             assertEquals(1.0D, rsp.getHits().getAt(2).field("foo").getValue(), 1.0D);
@@ -708,7 +708,7 @@ public void testBoolean() throws Exception {
         // ternary operator
         // vip's have a 50% discount
         assertNoFailuresAndResponse(buildRequest("doc['vip'] ? doc['price']/2 : doc['price']"), rsp -> {
-            assertEquals(3, rsp.getHits().getTotalHits().value);
+            assertEquals(3, rsp.getHits().getTotalHits().value());
             assertEquals(0.5D, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
             assertEquals(2.0D, rsp.getHits().getAt(1).field("foo").getValue(), 1.0D);
             assertEquals(2.0D, rsp.getHits().getAt(2).field("foo").getValue(), 1.0D);
@@ -727,7 +727,7 @@ public void testFilterScript() throws Exception {
         Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap());
         builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script)));
         assertNoFailuresAndResponse(builder, rsp -> {
-            assertEquals(1, rsp.getHits().getTotalHits().value);
+            assertEquals(1, rsp.getHits().getTotalHits().value());
             assertEquals(1.0D, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D);
         });
     }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java
index 0952ff8fe856f..bb714d4674ed6 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java
@@ -17,6 +17,8 @@
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.script.DoubleValuesScript;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.function.Function;
 
 /**
@@ -37,12 +39,20 @@ public DoubleValuesScript newInstance() {
         return new DoubleValuesScript() {
             @Override
             public double execute() {
-                return exprScript.evaluate(new DoubleValues[0]);
+                try {
+                    return exprScript.evaluate(new DoubleValues[0]);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
             }
 
             @Override
             public double evaluate(DoubleValues[] functionValues) {
-                return exprScript.evaluate(functionValues);
+                try {
+                    return exprScript.evaluate(functionValues);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
             }
 
             @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
index b306f104d7ba5..58cd9ea293aef 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.script.AggregationScript;
 import org.elasticsearch.script.BucketAggregationScript;
 import org.elasticsearch.script.BucketAggregationSelectorScript;
-import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.DoubleValuesScript;
 import org.elasticsearch.script.FieldScript;
 import org.elasticsearch.script.FilterScript;
@@ -36,9 +35,8 @@
 import org.elasticsearch.script.TermsSetQueryScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 
-import java.security.AccessControlContext;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -156,36 +154,14 @@ public String getType() {
 
     @Override
     public <T> T compile(String scriptName, String scriptSource, ScriptContext<T> context, Map<String, String> params) {
-        // classloader created here
-        final SecurityManager sm = System.getSecurityManager();
         SpecialPermission.check();
-        Expression expr = AccessController.doPrivileged(new PrivilegedAction<Expression>() {
-            @Override
-            public Expression run() {
-                try {
-                    // snapshot our context here, we check on behalf of the expression
-                    AccessControlContext engineContext = AccessController.getContext();
-                    ClassLoader loader = getClass().getClassLoader();
-                    if (sm != null) {
-                        loader = new ClassLoader(loader) {
-                            @Override
-                            protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
-                                try {
-                                    engineContext.checkPermission(new ClassPermission(name));
-                                } catch (SecurityException e) {
-                                    throw new ClassNotFoundException(name, e);
-                                }
-                                return super.loadClass(name, resolve);
-                            }
-                        };
-                    }
-                    // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here
-                    return JavascriptCompiler.compile(scriptSource, JavascriptCompiler.DEFAULT_FUNCTIONS, loader);
-                } catch (ParseException e) {
-                    throw convertToScriptException("compile error", scriptSource, scriptSource, e);
-                }
-            }
-        });
+        Expression expr;
+        try {
+            // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here
+            expr = JavascriptCompiler.compile(scriptSource, JavascriptCompiler.DEFAULT_FUNCTIONS);
+        } catch (ParseException e) {
+            throw convertToScriptException("compile error", scriptSource, scriptSource, e);
+        }
         if (contexts.containsKey(context) == false) {
             throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]");
         }
@@ -233,7 +209,11 @@ public Double execute() {
                             placeholder.setValue(((Number) value).doubleValue());
                         }
                     });
-                    return expr.evaluate(functionValuesArray);
+                    try {
+                        return expr.evaluate(functionValuesArray);
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
                 }
             };
         };
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
index c1cad60e94a9c..b748a0ced8be9 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
@@ -11,7 +11,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -46,9 +45,7 @@ public List<Route> routes() {
             new Route(GET, "/_msearch/template"),
             new Route(POST, "/_msearch/template"),
             new Route(GET, "/{index}/_msearch/template"),
-            new Route(POST, "/{index}/_msearch/template"),
-            Route.builder(GET, "/{index}/{type}/_msearch/template").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_msearch/template").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_msearch/template")
         );
     }
 
@@ -67,10 +64,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
      * Parses a {@link RestRequest} body and returns a {@link MultiSearchTemplateRequest}
      */
     public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
-        if (restRequest.getRestApiVersion() == RestApiVersion.V_7 && restRequest.hasParam("type")) {
-            restRequest.param("type");
-        }
-
         MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest();
         if (restRequest.hasParam("max_concurrent_searches")) {
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
index 152e98f39f3b4..50a130d0fbfe8 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
@@ -11,7 +11,6 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -48,13 +47,7 @@ public List<Route> routes() {
             new Route(GET, "/_search/template"),
             new Route(POST, "/_search/template"),
             new Route(GET, "/{index}/_search/template"),
-            new Route(POST, "/{index}/_search/template"),
-            Route.builder(GET, "/{index}/{type}/_search/template")
-                .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/{index}/{type}/_search/template")
-                .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7)
-                .build()
+            new Route(POST, "/{index}/_search/template")
         );
     }
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java
index 3efcfde684ebc..a3c0c60d75436 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java
@@ -138,7 +138,7 @@ protected void assertEqualInstances(SearchTemplateResponse expectedInstance, Sea
             SearchResponse expectedResponse = expectedInstance.getResponse();
             SearchResponse newResponse = newInstance.getResponse();
 
-            assertEquals(expectedResponse.getHits().getTotalHits().value, newResponse.getHits().getTotalHits().value);
+            assertEquals(expectedResponse.getHits().getTotalHits().value(), newResponse.getHits().getTotalHits().value());
             assertEquals(expectedResponse.getHits().getMaxScore(), newResponse.getHits().getMaxScore(), 0.0001);
         }
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
index fed598e46fbd9..cbb0e19d64a6e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
@@ -74,11 +74,6 @@ public void testMapBasic() throws IOException {
         Map<String, Object> state = new HashMap<>();
 
         Scorable scorer = new Scorable() {
-            @Override
-            public int docID() {
-                return 0;
-            }
-
             @Override
             public float score() {
                 return 0.5f;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java
index 01a9e995450aa..7edd6d5303252 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java
@@ -85,7 +85,7 @@ public void testBasics() throws IOException {
                     3.2f
                 );
                 TopDocs topDocs = searcher.search(query, 1);
-                assertEquals(1, topDocs.totalHits.value);
+                assertEquals(1, topDocs.totalHits.value());
                 assertEquals((float) (3.2 * 2 / 3), topDocs.scoreDocs[0].score, 0);
             }
         }
@@ -134,7 +134,7 @@ public void testWeightScript() throws IOException {
                     3.2f
                 );
                 TopDocs topDocs = searcher.search(query, 1);
-                assertEquals(1, topDocs.totalHits.value);
+                assertEquals(1, topDocs.totalHits.value());
                 assertEquals((float) (3.2 * 2 / 3), topDocs.scoreDocs[0].score, 0);
             }
         }
diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java
index 19173c650c24a..1c6ffe75e3fd2 100644
--- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java
+++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java
@@ -43,7 +43,7 @@ public void testRankFeaturesTermQuery() throws IOException {
         assertNoFailuresAndResponse(
             prepareSearch(INDEX_NAME).setQuery(QueryBuilders.termQuery(FIELD_NAME, HIGHER_RANKED_FEATURE)),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(2L));
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
                     assertThat(hit.getScore(), equalTo(20f));
                 }
@@ -52,7 +52,7 @@ public void testRankFeaturesTermQuery() throws IOException {
         assertNoFailuresAndResponse(
             prepareSearch(INDEX_NAME).setQuery(QueryBuilders.termQuery(FIELD_NAME, HIGHER_RANKED_FEATURE).boost(100f)),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(2L));
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
                     assertThat(hit.getScore(), equalTo(2000f));
                 }
@@ -67,7 +67,7 @@ public void testRankFeaturesTermQuery() throws IOException {
                     .minimumShouldMatch(1)
             ),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
                     if (hit.getId().equals("all")) {
                         assertThat(hit.getScore(), equalTo(50f));
@@ -83,7 +83,7 @@ public void testRankFeaturesTermQuery() throws IOException {
         );
         assertNoFailuresAndResponse(
             prepareSearch(INDEX_NAME).setQuery(QueryBuilders.termQuery(FIELD_NAME, "missing_feature")),
-            response -> assertThat(response.getHits().getTotalHits().value, equalTo(0L))
+            response -> assertThat(response.getHits().getTotalHits().value(), equalTo(0L))
         );
     }
 
diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
index 4fc4fc69e0ee8..97c97a643e9c8 100644
--- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
+++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
@@ -203,7 +203,7 @@ private SearchRequestBuilder prepareTokenCountFieldMapperSearch() {
     }
 
     private void assertSearchReturns(SearchResponse result, String... ids) {
-        assertThat(result.getHits().getTotalHits().value, equalTo((long) ids.length));
+        assertThat(result.getHits().getTotalHits().value(), equalTo((long) ids.length));
         assertThat(result.getHits().getHits().length, equalTo(ids.length));
         List<String> foundIds = new ArrayList<>();
         for (SearchHit hit : result.getHits()) {
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java
index bce6ffb5e0ea3..f277d28eed922 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java
@@ -468,8 +468,8 @@ public Query prefixQuery(
             }
             Automaton automaton = Operations.concatenate(automata);
             AutomatonQuery query = method == null
-                ? new AutomatonQuery(new Term(name(), value + "*"), automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false)
-                : new AutomatonQuery(new Term(name(), value + "*"), automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false, method);
+                ? new AutomatonQuery(new Term(name(), value + "*"), automaton, false)
+                : new AutomatonQuery(new Term(name(), value + "*"), automaton, false, method);
             return new BooleanQuery.Builder().add(query, BooleanClause.Occur.SHOULD)
                 .add(new TermQuery(new Term(parentField, value)), BooleanClause.Occur.SHOULD)
                 .build();
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java
index d16034c5de2fd..a992f68d93d9e 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java
@@ -34,6 +34,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.TwoPhaseIterator;
@@ -266,7 +267,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
 
             @Override
             public Explanation explain(LeafReaderContext context, int doc) throws IOException {
-                RuntimePhraseScorer scorer = scorer(context);
+                RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier(context).get(0);
                 if (scorer == null) {
                     return Explanation.noMatch("No matching phrase");
                 }
@@ -286,15 +287,26 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
             }
 
             @Override
-            public RuntimePhraseScorer scorer(LeafReaderContext context) throws IOException {
-                final Scorer approximationScorer = approximationWeight != null ? approximationWeight.scorer(context) : null;
-                if (approximationScorer == null) {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                ScorerSupplier approximationSupplier = approximationWeight != null ? approximationWeight.scorerSupplier(context) : null;
+                if (approximationSupplier == null) {
                     return null;
                 }
-                final DocIdSetIterator approximation = approximationScorer.iterator();
-                final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores());
-                final CheckedIntFunction<List<Object>, IOException> valueFetcher = valueFetcherProvider.apply(context);
-                return new RuntimePhraseScorer(this, approximation, leafSimScorer, valueFetcher, field, in);
+                return new ScorerSupplier() {
+                    @Override
+                    public Scorer get(long leadCost) throws IOException {
+                        final Scorer approximationScorer = approximationSupplier.get(leadCost);
+                        final DocIdSetIterator approximation = approximationScorer.iterator();
+                        final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores());
+                        final CheckedIntFunction<List<Object>, IOException> valueFetcher = valueFetcherProvider.apply(context);
+                        return new RuntimePhraseScorer(approximation, leafSimScorer, valueFetcher, field, in);
+                    }
+
+                    @Override
+                    public long cost() {
+                        return approximationSupplier.cost();
+                    }
+                };
             }
 
             @Override
@@ -310,7 +322,7 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException {
                     Weight innerWeight = in.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, 1);
                     return innerWeight.matches(context, doc);
                 }
-                RuntimePhraseScorer scorer = scorer(context);
+                RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier(context).get(0L);
                 if (scorer == null) {
                     return null;
                 }
@@ -336,14 +348,12 @@ private class RuntimePhraseScorer extends Scorer {
         private float freq;
 
         private RuntimePhraseScorer(
-            Weight weight,
             DocIdSetIterator approximation,
             LeafSimScorer scorer,
             CheckedIntFunction<List<Object>, IOException> valueFetcher,
             String field,
             Query query
         ) {
-            super(weight);
             this.scorer = scorer;
             this.valueFetcher = valueFetcher;
             this.field = field;
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java
index 922b92263d712..1eb6083cfe453 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java
@@ -89,8 +89,8 @@ private void assertPhraseQuery(MapperService mapperService) throws IOException {
                 SearchExecutionContext context = createSearchExecutionContext(mapperService, newSearcher(reader));
                 MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder("field", "brown fox");
                 TopDocs docs = context.searcher().search(queryBuilder.toQuery(context), 1);
-                assertThat(docs.totalHits.value, equalTo(1L));
-                assertThat(docs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+                assertThat(docs.totalHits.value(), equalTo(1L));
+                assertThat(docs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
                 assertThat(docs.scoreDocs[0].doc, equalTo(0));
             }
         }
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java
index 84139409e8bc6..a49e0c2a3e38d 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java
@@ -61,7 +61,7 @@ public class SourceConfirmedTextQueryTests extends ESTestCase {
     private static final IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> SOURCE_FETCHER_PROVIDER =
         context -> docID -> {
             sourceFetchCount.incrementAndGet();
-            return Collections.<Object>singletonList(context.reader().document(docID).get("body"));
+            return Collections.<Object>singletonList(context.reader().storedFields().document(docID).get("body"));
         };
 
     public void testTerm() throws Exception {
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java
index 0fef801b22009..2befcfb576017 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java
@@ -41,7 +41,7 @@
 public class SourceIntervalsSourceTests extends ESTestCase {
 
     private static final IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> SOURCE_FETCHER_PROVIDER =
-        context -> docID -> Collections.<Object>singletonList(context.reader().document(docID).get("body"));
+        context -> docID -> Collections.<Object>singletonList(context.reader().storedFields().document(docID).get("body"));
 
     public void testIntervals() throws IOException {
         final FieldType ft = new FieldType(TextField.TYPE_STORED);
diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java
index ad8e252e3fd63..9c0e5ce071dc6 100644
--- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java
+++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java
@@ -115,7 +115,7 @@ public void testParentWithMultipleBuckets() {
                     logger.info("bucket={}", bucket.getKey());
                     Children childrenBucket = bucket.getAggregations().get("to_comment");
                     TopHits topHits = childrenBucket.getAggregations().get("top_comments");
-                    logger.info("total_hits={}", topHits.getHits().getTotalHits().value);
+                    logger.info("total_hits={}", topHits.getHits().getTotalHits().value());
                     for (SearchHit searchHit : topHits.getHits()) {
                         logger.info("hit= {} {}", searchHit.getSortValues()[0], searchHit.getId());
                     }
@@ -129,7 +129,7 @@ public void testParentWithMultipleBuckets() {
                 assertThat(childrenBucket.getName(), equalTo("to_comment"));
                 assertThat(childrenBucket.getDocCount(), equalTo(2L));
                 TopHits topHits = childrenBucket.getAggregations().get("top_comments");
-                assertThat(topHits.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(topHits.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(topHits.getHits().getAt(0).getId(), equalTo("e"));
                 assertThat(topHits.getHits().getAt(1).getId(), equalTo("f"));
 
@@ -141,7 +141,7 @@ public void testParentWithMultipleBuckets() {
                 assertThat(childrenBucket.getName(), equalTo("to_comment"));
                 assertThat(childrenBucket.getDocCount(), equalTo(1L));
                 topHits = childrenBucket.getAggregations().get("top_comments");
-                assertThat(topHits.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(topHits.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(topHits.getHits().getAt(0).getId(), equalTo("f"));
 
                 categoryBucket = categoryTerms.getBucketByKey("c");
@@ -152,7 +152,7 @@ public void testParentWithMultipleBuckets() {
                 assertThat(childrenBucket.getName(), equalTo("to_comment"));
                 assertThat(childrenBucket.getDocCount(), equalTo(1L));
                 topHits = childrenBucket.getAggregations().get("top_comments");
-                assertThat(topHits.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(topHits.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(topHits.getHits().getAt(0).getId(), equalTo("f"));
             }
         );
diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
index 872165014f5a4..cce0ef06cbf62 100644
--- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
+++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
@@ -107,7 +107,7 @@ public void testMultiLevelChild() throws Exception {
                     )
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
             }
         );
@@ -117,7 +117,7 @@ public void testMultiLevelChild() throws Exception {
                 boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", termQuery("p_field", "p_value1"), false))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("c1"));
             }
         );
@@ -127,7 +127,7 @@ public void testMultiLevelChild() throws Exception {
                 boolQuery().must(matchAllQuery()).filter(hasParentQuery("child", termQuery("c_field", "c_value1"), false))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("gc1"));
             }
         );
@@ -135,7 +135,7 @@ public void testMultiLevelChild() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasParentQuery("parent", termQuery("p_field", "p_value1"), false)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("c1"));
             }
         );
@@ -143,7 +143,7 @@ public void testMultiLevelChild() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasParentQuery("child", termQuery("c_field", "c_value1"), false)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("gc1"));
             }
         );
@@ -161,7 +161,7 @@ public void test2744() throws IOException {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasChildQuery("test", matchQuery("foo", 1), ScoreMode.None)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
             }
         );
@@ -182,7 +182,7 @@ public void testSimpleChildQuery() throws Exception {
 
         // TEST FETCHING _parent from child
         assertNoFailuresAndResponse(prepareSearch("test").setQuery(idsQuery().addIds("c1")), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getAt(0).getId(), equalTo("c1"));
             assertThat(extractValue("join_field.name", response.getHits().getAt(0).getSourceAsMap()), equalTo("child"));
             assertThat(extractValue("join_field.parent", response.getHits().getAt(0).getSourceAsMap()), equalTo("p1"));
@@ -195,7 +195,7 @@ public void testSimpleChildQuery() throws Exception {
                 boolQuery().filter(termQuery("join_field#parent", "p1")).filter(termQuery("join_field", "child"))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("c1"), equalTo("c2")));
                 assertThat(extractValue("join_field.name", response.getHits().getAt(0).getSourceAsMap()), equalTo("child"));
                 assertThat(extractValue("join_field.parent", response.getHits().getAt(0).getSourceAsMap()), equalTo("p1"));
@@ -208,7 +208,7 @@ public void testSimpleChildQuery() throws Exception {
         // HAS CHILD
         assertNoFailuresAndResponse(prepareSearch("test").setQuery(randomHasChild("child", "c_field", "yellow")), response -> {
             assertHitCount(response, 1L);
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
         });
 
@@ -307,8 +307,8 @@ public void testHasParentFilter() throws Exception {
                 ).setSize(numChildDocsPerParent),
                 response -> {
                     Set<String> childIds = parentToChildrenEntry.getValue();
-                    assertThat(response.getHits().getTotalHits().value, equalTo((long) childIds.size()));
-                    for (int i = 0; i < response.getHits().getTotalHits().value; i++) {
+                    assertThat(response.getHits().getTotalHits().value(), equalTo((long) childIds.size()));
+                    for (int i = 0; i < response.getHits().getTotalHits().value(); i++) {
                         assertThat(childIds.remove(response.getHits().getAt(i).getId()), is(true));
                         assertThat(response.getHits().getAt(i).getScore(), is(1.0f));
                     }
@@ -341,7 +341,7 @@ public void testSimpleChildQueryWithFlush() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
             }
         );
@@ -349,7 +349,7 @@ public void testSimpleChildQueryWithFlush() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p2"));
             }
         );
@@ -357,7 +357,7 @@ public void testSimpleChildQueryWithFlush() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasChildQuery("child", termQuery("c_field", "red"), ScoreMode.None)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("p2"), equalTo("p1")));
                 assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("p2"), equalTo("p1")));
             }
@@ -367,7 +367,7 @@ public void testSimpleChildQueryWithFlush() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
             }
         );
@@ -375,7 +375,7 @@ public void testSimpleChildQueryWithFlush() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "blue"), ScoreMode.None))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p2"));
             }
         );
@@ -383,7 +383,7 @@ public void testSimpleChildQueryWithFlush() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "red"), ScoreMode.None))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("p2"), equalTo("p1")));
                 assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("p2"), equalTo("p1")));
             }
@@ -426,7 +426,7 @@ public void testScopedFacet() throws Exception {
                         )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("p2"), equalTo("p1")));
                 assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("p2"), equalTo("p1")));
 
@@ -458,7 +458,7 @@ public void testDeletedParent() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
                 assertThat(response.getHits().getAt(0).getSourceAsString(), containsString("\"p_value1\""));
             }
@@ -472,7 +472,7 @@ public void testDeletedParent() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.None))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
                 assertThat(response.getHits().getAt(0).getSourceAsString(), containsString("\"p_value1_updated\""));
             }
@@ -647,7 +647,7 @@ public void testScoreForParentChildQueriesWithFunctionScore() throws Exception {
                 )
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("1"));
                 assertThat(response.getHits().getHits()[0].getScore(), equalTo(6f));
                 assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -667,7 +667,7 @@ public void testScoreForParentChildQueriesWithFunctionScore() throws Exception {
                 )
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
                 assertThat(response.getHits().getHits()[0].getScore(), equalTo(4f));
                 assertThat(response.getHits().getHits()[1].getId(), equalTo("2"));
@@ -687,7 +687,7 @@ public void testScoreForParentChildQueriesWithFunctionScore() throws Exception {
                 )
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
                 assertThat(response.getHits().getHits()[0].getScore(), equalTo(4f));
                 assertThat(response.getHits().getHits()[1].getId(), equalTo("2"));
@@ -707,7 +707,7 @@ public void testScoreForParentChildQueriesWithFunctionScore() throws Exception {
                 )
             ).addSort(SortBuilders.fieldSort("c_field3")).addSort(SortBuilders.scoreSort()),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("16"));
                 assertThat(response.getHits().getHits()[0].getScore(), equalTo(5f));
                 assertThat(response.getHits().getHits()[1].getId(), equalTo("17"));
@@ -768,7 +768,7 @@ public void testHasChildAndHasParentFilter_withFilter() throws Exception {
                 boolQuery().must(matchAllQuery()).filter(hasChildQuery("child", termQuery("c_field", 1), ScoreMode.None))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("1"));
             }
         );
@@ -778,7 +778,7 @@ public void testHasChildAndHasParentFilter_withFilter() throws Exception {
                 boolQuery().must(matchAllQuery()).filter(hasParentQuery("parent", termQuery("p_field", 1), false))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("2"));
             }
         );
@@ -801,7 +801,7 @@ public void testHasChildInnerHitsHighlighting() throws Exception {
                 )
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("1"));
                 SearchHit[] searchHits = response.getHits().getHits()[0].getInnerHits().get("child").getHits();
                 assertThat(searchHits.length, equalTo(1));
@@ -888,7 +888,7 @@ public void testSimpleQueryRewrite() throws Exception {
                     .addSort("p_field", SortOrder.ASC)
                     .setSize(5),
                 response -> {
-                    assertThat(response.getHits().getTotalHits().value, equalTo(10L));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo(10L));
                     assertThat(response.getHits().getHits()[0].getId(), equalTo("p000"));
                     assertThat(response.getHits().getHits()[1].getId(), equalTo("p001"));
                     assertThat(response.getHits().getHits()[2].getId(), equalTo("p002"));
@@ -903,7 +903,7 @@ public void testSimpleQueryRewrite() throws Exception {
                     .addSort("c_field", SortOrder.ASC)
                     .setSize(5),
                 response -> {
-                    assertThat(response.getHits().getTotalHits().value, equalTo(500L));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo(500L));
                     assertThat(response.getHits().getHits()[0].getId(), equalTo("c000"));
                     assertThat(response.getHits().getHits()[1].getId(), equalTo("c001"));
                     assertThat(response.getHits().getHits()[2].getId(), equalTo("c002"));
@@ -932,7 +932,7 @@ public void testReIndexingParentAndChildDocuments() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.Total)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
                 assertThat(response.getHits().getAt(0).getSourceAsString(), containsString("\"p_value1\""));
             }
@@ -943,7 +943,7 @@ public void testReIndexingParentAndChildDocuments() throws Exception {
                 boolQuery().must(matchQuery("c_field", "x")).must(hasParentQuery("parent", termQuery("p_field", "p_value2"), true))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("c3"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("c4"));
             }
@@ -961,7 +961,7 @@ public void testReIndexingParentAndChildDocuments() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasChildQuery("child", termQuery("c_field", "yellow"), ScoreMode.Total)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p1"));
                 assertThat(response.getHits().getAt(0).getSourceAsString(), containsString("\"p_value1\""));
             }
@@ -972,7 +972,7 @@ public void testReIndexingParentAndChildDocuments() throws Exception {
                 boolQuery().must(matchQuery("c_field", "x")).must(hasParentQuery("parent", termQuery("p_field", "p_value2"), true))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getId(), Matchers.anyOf(equalTo("c3"), equalTo("c4")));
                 assertThat(response.getHits().getAt(1).getId(), Matchers.anyOf(equalTo("c3"), equalTo("c4")));
             }
@@ -996,7 +996,7 @@ public void testHasChildQueryWithMinimumScore() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.Total)).setMinScore(3),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("p2"));
                 assertThat(response.getHits().getAt(0).getScore(), equalTo(3.0f));
             }
@@ -1411,7 +1411,7 @@ public void testParentChildQueriesViaScrollApi() throws Exception {
                 10,
                 (respNum, response) -> {
                     assertNoFailures(response);
-                    assertThat(response.getHits().getTotalHits().value, equalTo(10L));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo(10L));
                 }
             );
         }
@@ -1469,7 +1469,7 @@ public void testMinMaxChildren() throws Exception {
 
         // Score mode = NONE
         assertResponse(minMaxQuery(ScoreMode.None, 1, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("2"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1479,7 +1479,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.None, 2, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("4"));
@@ -1487,7 +1487,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.None, 3, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1f));
         });
@@ -1495,7 +1495,7 @@ public void testMinMaxChildren() throws Exception {
         assertHitCount(minMaxQuery(ScoreMode.None, 4, null), 0L);
 
         assertResponse(minMaxQuery(ScoreMode.None, 1, 4), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("2"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1505,7 +1505,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.None, 1, 3), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("2"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1515,7 +1515,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.None, 1, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("2"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1523,7 +1523,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.None, 2, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1f));
         });
@@ -1533,7 +1533,7 @@ public void testMinMaxChildren() throws Exception {
 
         // Score mode = SUM
         assertResponse(minMaxQuery(ScoreMode.Total, 1, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(6f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1543,7 +1543,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Total, 2, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(6f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1551,7 +1551,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Total, 3, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(6f));
         });
@@ -1559,7 +1559,7 @@ public void testMinMaxChildren() throws Exception {
         assertHitCount(minMaxQuery(ScoreMode.Total, 4, null), 0L);
 
         assertResponse(minMaxQuery(ScoreMode.Total, 1, 4), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(6f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1569,7 +1569,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Total, 1, 3), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(6f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1579,7 +1579,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Total, 1, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(3f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("2"));
@@ -1587,7 +1587,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Total, 2, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(3f));
         });
@@ -1597,7 +1597,7 @@ public void testMinMaxChildren() throws Exception {
 
         // Score mode = MAX
         assertResponse(minMaxQuery(ScoreMode.Max, 1, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(3f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1607,7 +1607,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Max, 2, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(3f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1615,7 +1615,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Max, 3, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(3f));
         });
@@ -1623,7 +1623,7 @@ public void testMinMaxChildren() throws Exception {
         assertHitCount(minMaxQuery(ScoreMode.Max, 4, null), 0L);
 
         assertResponse(minMaxQuery(ScoreMode.Max, 1, 4), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(3f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1633,7 +1633,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Max, 1, 3), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(3f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1643,7 +1643,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Max, 1, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(2f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("2"));
@@ -1651,7 +1651,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Max, 2, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(2f));
         });
@@ -1661,7 +1661,7 @@ public void testMinMaxChildren() throws Exception {
 
         // Score mode = AVG
         assertResponse(minMaxQuery(ScoreMode.Avg, 1, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(2f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1671,7 +1671,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Avg, 2, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(2f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1679,7 +1679,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Avg, 3, null), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(2f));
         });
@@ -1687,7 +1687,7 @@ public void testMinMaxChildren() throws Exception {
         assertHitCount(minMaxQuery(ScoreMode.Avg, 4, null), 0L);
 
         assertResponse(minMaxQuery(ScoreMode.Avg, 1, 4), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(2f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1697,7 +1697,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Avg, 1, 3), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("4"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(2f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -1707,7 +1707,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Avg, 1, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1.5f));
             assertThat(response.getHits().getHits()[1].getId(), equalTo("2"));
@@ -1715,7 +1715,7 @@ public void testMinMaxChildren() throws Exception {
         });
 
         assertResponse(minMaxQuery(ScoreMode.Avg, 2, 2), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits()[0].getId(), equalTo("3"));
             assertThat(response.getHits().getHits()[0].getScore(), equalTo(1.5f));
         });
diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java
index 0ae10b297f709..6d6072b2992ca 100644
--- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java
+++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java
@@ -128,7 +128,7 @@ public void testSimpleParentChild() throws Exception {
 
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-                assertThat(innerHits.getTotalHits().value, equalTo(2L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(2L));
 
                 assertThat(innerHits.getAt(0).getId(), equalTo("c1"));
                 assertThat(innerHits.getAt(1).getId(), equalTo("c2"));
@@ -148,7 +148,7 @@ public void testSimpleParentChild() throws Exception {
 
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-                assertThat(innerHits.getTotalHits().value, equalTo(3L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(3L));
 
                 assertThat(innerHits.getAt(0).getId(), equalTo("c4"));
                 assertThat(innerHits.getAt(1).getId(), equalTo("c5"));
@@ -280,7 +280,7 @@ public void testRandomParentChild() throws Exception {
                 assertThat(searchHit.getShard(), notNullValue());
 
                 SearchHits inner = searchHit.getInnerHits().get("a");
-                assertThat(inner.getTotalHits().value, equalTo((long) child1InnerObjects[parent]));
+                assertThat(inner.getTotalHits().value(), equalTo((long) child1InnerObjects[parent]));
                 for (int child = 0; child < child1InnerObjects[parent] && child < size; child++) {
                     SearchHit innerHit = inner.getAt(child);
                     String childId = String.format(Locale.ENGLISH, "c1_%04d", offset1 + child);
@@ -290,7 +290,7 @@ public void testRandomParentChild() throws Exception {
                 offset1 += child1InnerObjects[parent];
 
                 inner = searchHit.getInnerHits().get("b");
-                assertThat(inner.getTotalHits().value, equalTo((long) child2InnerObjects[parent]));
+                assertThat(inner.getTotalHits().value(), equalTo((long) child2InnerObjects[parent]));
                 for (int child = 0; child < child2InnerObjects[parent] && child < size; child++) {
                     SearchHit innerHit = inner.getAt(child);
                     String childId = String.format(Locale.ENGLISH, "c2_%04d", offset2 + child);
@@ -347,12 +347,12 @@ public void testInnerHitsOnHasParent() throws Exception {
 
                 SearchHit searchHit = response.getHits().getAt(0);
                 assertThat(searchHit.getId(), equalTo("3"));
-                assertThat(searchHit.getInnerHits().get("question").getTotalHits().value, equalTo(1L));
+                assertThat(searchHit.getInnerHits().get("question").getTotalHits().value(), equalTo(1L));
                 assertThat(searchHit.getInnerHits().get("question").getAt(0).getId(), equalTo("1"));
 
                 searchHit = response.getHits().getAt(1);
                 assertThat(searchHit.getId(), equalTo("4"));
-                assertThat(searchHit.getInnerHits().get("question").getTotalHits().value, equalTo(1L));
+                assertThat(searchHit.getInnerHits().get("question").getTotalHits().value(), equalTo(1L));
                 assertThat(searchHit.getInnerHits().get("question").getAt(0).getId(), equalTo("2"));
             }
         );
@@ -394,11 +394,11 @@ public void testParentChildMultipleLayers() throws Exception {
 
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getAt(0).getId(), equalTo("3"));
 
                 innerHits = innerHits.getAt(0).getInnerHits().get("remark");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getAt(0).getId(), equalTo("5"));
             }
         );
@@ -417,11 +417,11 @@ public void testParentChildMultipleLayers() throws Exception {
 
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getAt(0).getId(), equalTo("4"));
 
                 innerHits = innerHits.getAt(0).getInnerHits().get("remark");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getAt(0).getId(), equalTo("6"));
             }
         );
@@ -482,34 +482,34 @@ public void testRoyals() throws Exception {
                 assertThat(response.getHits().getAt(0).getId(), equalTo("duke"));
 
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("earls");
-                assertThat(innerHits.getTotalHits().value, equalTo(4L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(4L));
                 assertThat(innerHits.getAt(0).getId(), equalTo("earl1"));
                 assertThat(innerHits.getAt(1).getId(), equalTo("earl2"));
                 assertThat(innerHits.getAt(2).getId(), equalTo("earl3"));
                 assertThat(innerHits.getAt(3).getId(), equalTo("earl4"));
 
                 SearchHits innerInnerHits = innerHits.getAt(0).getInnerHits().get("barons");
-                assertThat(innerInnerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerInnerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron1"));
 
                 innerInnerHits = innerHits.getAt(1).getInnerHits().get("barons");
-                assertThat(innerInnerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerInnerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron2"));
 
                 innerInnerHits = innerHits.getAt(2).getInnerHits().get("barons");
-                assertThat(innerInnerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerInnerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron3"));
 
                 innerInnerHits = innerHits.getAt(3).getInnerHits().get("barons");
-                assertThat(innerInnerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerInnerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerInnerHits.getAt(0).getId(), equalTo("baron4"));
 
                 innerHits = response.getHits().getAt(0).getInnerHits().get("princes");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getAt(0).getId(), equalTo("prince"));
 
                 innerInnerHits = innerHits.getAt(0).getInnerHits().get("kings");
-                assertThat(innerInnerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerInnerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerInnerHits.getAt(0).getId(), equalTo("king"));
             }
         );
@@ -532,12 +532,12 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception {
             response -> {
                 assertHitCount(response, 2);
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
-                assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));
 
                 assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
-                assertThat(response.getHits().getAt(1).getInnerHits().get("child").getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getAt(1).getInnerHits().get("child").getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
                 assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));
             }
@@ -549,7 +549,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception {
         assertResponse(prepareSearch("index").setQuery(query).addSort("id", SortOrder.ASC), response -> {
             assertHitCount(response, 1);
             assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
-            assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getAt(0).getInnerHits().get("child").getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2"));
         });
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java
index 258cbe743d7d3..60412179807a5 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java
@@ -102,7 +102,7 @@ public final LeafBucketCollector getLeafCollector(AggregationExecutionContext ag
             public void collect(int docId, long owningBucketOrd) throws IOException {
                 if (parentDocs.get(docId) && globalOrdinals.advanceExact(docId)) {
                     int globalOrdinal = (int) globalOrdinals.nextOrd();
-                    assert globalOrdinal != -1 && globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS;
+                    assert globalOrdinal != -1 && globalOrdinals.docValueCount() == 1;
                     collectionStrategy.add(owningBucketOrd, globalOrdinal);
                 }
             }
@@ -134,11 +134,6 @@ protected void prepareSubAggs(long[] ordsToCollect) throws IOException {
                 public float score() {
                     return 1f;
                 }
-
-                @Override
-                public int docID() {
-                    return childDocsIter.docID();
-                }
             });
 
             final Bits liveDocs = ctx.reader().getLiveDocs();
@@ -150,7 +145,7 @@ public int docID() {
                     continue;
                 }
                 int globalOrdinal = (int) globalOrdinals.nextOrd();
-                assert globalOrdinal != -1 && globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS;
+                assert globalOrdinal != -1 && globalOrdinals.docValueCount() == 1;
                 /*
                  * Check if we contain every ordinal. It's almost certainly be
                  * faster to replay all the matching ordinals and filter them down
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
index 9ecf4ed821e2a..6b00e94431bef 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
@@ -20,8 +20,8 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopDocsCollector;
-import org.apache.lucene.search.TopFieldCollector;
-import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.search.TopFieldCollectorManager;
+import org.apache.lucene.search.TopScoreDocCollectorManager;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.search.Weight;
@@ -137,12 +137,12 @@ public TopDocsAndMaxScore topDocs(SearchHit hit) throws IOException {
                 TopDocsCollector<?> topDocsCollector;
                 MaxScoreCollector maxScoreCollector = null;
                 if (sort() != null) {
-                    topDocsCollector = TopFieldCollector.create(sort().sort, topN, Integer.MAX_VALUE);
+                    topDocsCollector = new TopFieldCollectorManager(sort().sort, topN, null, Integer.MAX_VALUE, false).newCollector();
                     if (trackScores()) {
                         maxScoreCollector = new MaxScoreCollector();
                     }
                 } else {
-                    topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
+                    topDocsCollector = new TopScoreDocCollectorManager(topN, null, Integer.MAX_VALUE, false).newCollector();
                     maxScoreCollector = new MaxScoreCollector();
                 }
                 for (LeafReaderContext ctx : this.context.searcher().getIndexReader().leaves()) {
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java
index 03a1677e60f47..707fcc822665f 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java
@@ -107,7 +107,7 @@ public void testParentChild() throws IOException {
 
         // verify for each children
         for (String parent : expectedParentChildRelations.keySet()) {
-            testCase(new TermInSetQuery(IdFieldMapper.NAME, Uid.encodeId("child0_" + parent)), indexReader, aggregation -> {
+            testCase(new TermInSetQuery(IdFieldMapper.NAME, List.of(Uid.encodeId("child0_" + parent))), indexReader, aggregation -> {
                 assertEquals(
                     "Expected one result for min-aggregation for parent: " + parent + ", but had aggregation-results: " + aggregation,
                     1,
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java
index 91ec0e3c67691..ca90b0e588b18 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java
@@ -104,7 +104,7 @@ public void testParentChild() throws IOException {
         });
 
         for (String parent : expectedParentChildRelations.keySet()) {
-            testCase(new TermInSetQuery(IdFieldMapper.NAME, Uid.encodeId(parent)), indexReader, child -> {
+            testCase(new TermInSetQuery(IdFieldMapper.NAME, List.of(Uid.encodeId(parent))), indexReader, child -> {
                 assertEquals((long) expectedParentChildRelations.get(parent).v1(), child.getDocCount());
                 assertEquals(
                     expectedParentChildRelations.get(parent).v2(),
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
index d4fe49ec8c773..9244f815cd957 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
@@ -54,6 +54,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
@@ -341,13 +342,13 @@ static void assertLateParsingQuery(Query query, String type, String id) throws I
         BooleanQuery booleanQuery = (BooleanQuery) lateParsingQuery.getInnerQuery();
         assertThat(booleanQuery.clauses().size(), equalTo(2));
         // check the inner ids query, we have to call rewrite to get to check the type it's executed against
-        assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST));
-        assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
-        TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery();
-        assertEquals(new TermInSetQuery(IdFieldMapper.NAME, Uid.encodeId(id)), termsQuery);
+        assertThat(booleanQuery.clauses().get(0).occur(), equalTo(BooleanClause.Occur.MUST));
+        assertThat(booleanQuery.clauses().get(0).query(), instanceOf(TermInSetQuery.class));
+        TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).query();
+        assertEquals(new TermInSetQuery(IdFieldMapper.NAME, List.of(Uid.encodeId(id))), termsQuery);
         // check the type filter
-        assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
-        assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery());
+        assertThat(booleanQuery.clauses().get(1).occur(), equalTo(BooleanClause.Occur.FILTER));
+        assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).query());
     }
 
     @Override
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 255131b51a57a..393c7b6157077 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -110,74 +110,93 @@ public Explanation explain(LeafReaderContext leafReaderContext, int docId) throw
             }
 
             @Override
-            public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
-                final Scorer approximation = candidateMatchesWeight.scorer(leafReaderContext);
-                if (approximation == null) {
+            public ScorerSupplier scorerSupplier(LeafReaderContext leafReaderContext) throws IOException {
+                final ScorerSupplier approximationSupplier = candidateMatchesWeight.scorerSupplier(leafReaderContext);
+                if (approximationSupplier == null) {
                     return null;
                 }
 
-                final CheckedFunction<Integer, Query, IOException> percolatorQueries = queryStore.getQueries(leafReaderContext);
+                ScorerSupplier verifiedDocsScorer;
                 if (scoreMode.needsScores()) {
-                    return new BaseScorer(this, approximation) {
-
-                        float score;
-
-                        @Override
-                        boolean matchDocId(int docId) throws IOException {
-                            Query query = percolatorQueries.apply(docId);
-                            if (query != null) {
-                                if (nonNestedDocsFilter != null) {
-                                    query = new BooleanQuery.Builder().add(query, Occur.MUST)
-                                        .add(nonNestedDocsFilter, Occur.FILTER)
-                                        .build();
-                                }
-                                TopDocs topDocs = percolatorIndexSearcher.search(query, 1);
-                                if (topDocs.scoreDocs.length > 0) {
-                                    score = topDocs.scoreDocs[0].score;
-                                    return true;
-                                } else {
-                                    return false;
+                    verifiedDocsScorer = null;
+                } else {
+                    verifiedDocsScorer = verifiedMatchesWeight.scorerSupplier(leafReaderContext);
+                }
+
+                return new ScorerSupplier() {
+                    @Override
+                    public Scorer get(long leadCost) throws IOException {
+                        final Scorer approximation = approximationSupplier.get(leadCost);
+                        final CheckedFunction<Integer, Query, IOException> percolatorQueries = queryStore.getQueries(leafReaderContext);
+                        if (scoreMode.needsScores()) {
+                            return new BaseScorer(approximation) {
+
+                                float score;
+
+                                @Override
+                                boolean matchDocId(int docId) throws IOException {
+                                    Query query = percolatorQueries.apply(docId);
+                                    if (query != null) {
+                                        if (nonNestedDocsFilter != null) {
+                                            query = new BooleanQuery.Builder().add(query, Occur.MUST)
+                                                .add(nonNestedDocsFilter, Occur.FILTER)
+                                                .build();
+                                        }
+                                        TopDocs topDocs = percolatorIndexSearcher.search(query, 1);
+                                        if (topDocs.scoreDocs.length > 0) {
+                                            score = topDocs.scoreDocs[0].score;
+                                            return true;
+                                        } else {
+                                            return false;
+                                        }
+                                    } else {
+                                        return false;
+                                    }
                                 }
-                            } else {
-                                return false;
-                            }
-                        }
 
-                        @Override
-                        public float score() {
-                            return score;
-                        }
-                    };
-                } else {
-                    ScorerSupplier verifiedDocsScorer = verifiedMatchesWeight.scorerSupplier(leafReaderContext);
-                    Bits verifiedDocsBits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), verifiedDocsScorer);
-                    return new BaseScorer(this, approximation) {
+                                @Override
+                                public float score() {
+                                    return score;
+                                }
+                            };
+                        } else {
+                            Bits verifiedDocsBits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), verifiedDocsScorer);
+                            return new BaseScorer(approximation) {
+
+                                @Override
+                                public float score() throws IOException {
+                                    return 0f;
+                                }
 
-                        @Override
-                        public float score() throws IOException {
-                            return 0f;
+                                boolean matchDocId(int docId) throws IOException {
+                                    // We use the verifiedDocsBits to skip the expensive MemoryIndex verification.
+                                    // If docId also appears in the verifiedDocsBits then that means during indexing
+                                    // we were able to extract all query terms and for this candidate match
+                                    // and we determined based on the nature of the query that it is safe to skip
+                                    // the MemoryIndex verification.
+                                    if (verifiedDocsBits.get(docId)) {
+                                        return true;
+                                    }
+                                    Query query = percolatorQueries.apply(docId);
+                                    if (query == null) {
+                                        return false;
+                                    }
+                                    if (nonNestedDocsFilter != null) {
+                                        query = new BooleanQuery.Builder().add(query, Occur.MUST)
+                                            .add(nonNestedDocsFilter, Occur.FILTER)
+                                            .build();
+                                    }
+                                    return Lucene.exists(percolatorIndexSearcher, query);
+                                }
+                            };
                         }
+                    }
 
-                        boolean matchDocId(int docId) throws IOException {
-                            // We use the verifiedDocsBits to skip the expensive MemoryIndex verification.
-                            // If docId also appears in the verifiedDocsBits then that means during indexing
-                            // we were able to extract all query terms and for this candidate match
-                            // and we determined based on the nature of the query that it is safe to skip
-                            // the MemoryIndex verification.
-                            if (verifiedDocsBits.get(docId)) {
-                                return true;
-                            }
-                            Query query = percolatorQueries.apply(docId);
-                            if (query == null) {
-                                return false;
-                            }
-                            if (nonNestedDocsFilter != null) {
-                                query = new BooleanQuery.Builder().add(query, Occur.MUST).add(nonNestedDocsFilter, Occur.FILTER).build();
-                            }
-                            return Lucene.exists(percolatorIndexSearcher, query);
-                        }
-                    };
-                }
+                    @Override
+                    public long cost() {
+                        return approximationSupplier.cost();
+                    }
+                };
             }
 
             @Override
@@ -265,8 +284,7 @@ abstract static class BaseScorer extends Scorer {
 
         final Scorer approximation;
 
-        BaseScorer(Weight weight, Scorer approximation) {
-            super(weight);
+        BaseScorer(Scorer approximation) {
             this.approximation = approximation;
         }
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 0ee93474234ec..85af5b120f6fd 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -46,7 +46,6 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -54,7 +53,6 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
@@ -85,7 +83,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.core.RestApiVersion.equalTo;
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -312,9 +309,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             if (indexedDocumentIndex != null) {
                 builder.field(INDEXED_DOCUMENT_FIELD_INDEX.getPreferredName(), indexedDocumentIndex);
             }
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(INDEXED_DOCUMENT_FIELD_TYPE.getPreferredName(), MapperService.SINGLE_MAPPING_NAME);
-            }
             if (indexedDocumentId != null) {
                 builder.field(INDEXED_DOCUMENT_FIELD_ID.getPreferredName(), indexedDocumentId);
             }
@@ -372,14 +366,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             DOCUMENTS_FIELD.getPreferredName(),
             INDEXED_DOCUMENT_FIELD_ID.getPreferredName()
         );
-        PARSER.declareString(
-            deprecateAndIgnoreType("percolate_with_type", TYPE_DEPRECATION_MESSAGE),
-            INDEXED_DOCUMENT_FIELD_TYPE.forRestApiVersion(equalTo(RestApiVersion.V_7))
-        );
-        PARSER.declareString(
-            deprecateAndIgnoreType("percolate_with_document_type", DOCUMENT_TYPE_DEPRECATION_MESSAGE),
-            DOCUMENT_TYPE_FIELD.forRestApiVersion(equalTo(RestApiVersion.V_7))
-        );
     }
 
     private static BiConsumer<PercolateQueryBuilder, String> deprecateAndIgnoreType(String key, String message) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 409b6fd70c3c7..d6422efdfed26 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -294,7 +294,7 @@ Tuple<BooleanQuery, Boolean> createCandidateQuery(IndexReader indexReader) throw
             List<BytesRef> extractedTerms = t.v1();
             Map<String, List<byte[]>> encodedPointValuesByField = t.v2();
             // `1 + ` is needed to take into account the EXTRACTION_FAILED should clause
-            boolean canUseMinimumShouldMatchField = 1 + extractedTerms.size() + encodedPointValuesByField.size() <= BooleanQuery
+            boolean canUseMinimumShouldMatchField = 1 + extractedTerms.size() + encodedPointValuesByField.size() <= IndexSearcher
                 .getMaxClauseCount();
 
             List<Query> subQueries = new ArrayList<>();
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java
index c363746856681..8413b564c2041 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java
@@ -91,7 +91,7 @@ public void process(HitContext hitContext) throws IOException {
                     query = percolatorIndexSearcher.rewrite(query);
                     int memoryIndexMaxDoc = percolatorIndexSearcher.getIndexReader().maxDoc();
                     TopDocs topDocs = percolatorIndexSearcher.search(query, memoryIndexMaxDoc, new Sort(SortField.FIELD_DOC));
-                    if (topDocs.totalHits.value == 0) {
+                    if (topDocs.totalHits.value() == 0) {
                         // This hit didn't match with a percolate query,
                         // likely to happen when percolating multiple documents
                         continue;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
index da4b10956dcf8..0e9aa6de3a0c0 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
@@ -8,7 +8,6 @@
  */
 package org.elasticsearch.percolator;
 
-import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.spans.SpanOrQuery;
 import org.apache.lucene.queries.spans.SpanTermQuery;
@@ -26,12 +25,15 @@
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.index.query.DateRangeIncludingNowQuery;
 import org.elasticsearch.lucene.queries.BlendedTermQuery;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -162,7 +164,7 @@ public QueryVisitor getSubVisitor(Occur occur, Query parent) {
             int minimumShouldMatchValue = 0;
             if (parent instanceof BooleanQuery bq) {
                 if (bq.getMinimumNumberShouldMatch() == 0
-                    && bq.clauses().stream().anyMatch(c -> c.getOccur() == Occur.MUST || c.getOccur() == Occur.FILTER)) {
+                    && bq.clauses().stream().anyMatch(c -> c.occur() == Occur.MUST || c.occur() == Occur.FILTER)) {
                     return QueryVisitor.EMPTY_VISITOR;
                 }
                 minimumShouldMatchValue = bq.getMinimumNumberShouldMatch();
@@ -198,11 +200,15 @@ public void consumeTerms(Query query, Term... termsToConsume) {
         @Override
         public void consumeTermsMatching(Query query, String field, Supplier<ByteRunAutomaton> automaton) {
             if (query instanceof TermInSetQuery q) {
-                PrefixCodedTerms.TermIterator ti = q.getTermData().iterator();
+                BytesRefIterator bytesRefIterator = q.getBytesRefIterator();
                 BytesRef term;
                 Set<QueryExtraction> qe = new HashSet<>();
-                while ((term = ti.next()) != null) {
-                    qe.add(new QueryExtraction(new Term(field, term)));
+                try {
+                    while ((term = bytesRefIterator.next()) != null) {
+                        qe.add(new QueryExtraction(new Term(field, term)));
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
                 }
                 this.terms.add(new Result(true, qe, 1));
             } else {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
index 31e893ace72fd..ff321303b56c0 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
@@ -31,6 +31,7 @@
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.memory.MemoryIndex;
@@ -56,6 +57,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermInSetQuery;
@@ -246,15 +248,13 @@ public void testDuel() throws Exception {
             queryFunctions.add(
                 () -> new TermInSetQuery(
                     field1,
-                    new BytesRef(randomFrom(stringContent.get(field1))),
-                    new BytesRef(randomFrom(stringContent.get(field1)))
+                    List.of(new BytesRef(randomFrom(stringContent.get(field1))), new BytesRef(randomFrom(stringContent.get(field1))))
                 )
             );
             queryFunctions.add(
                 () -> new TermInSetQuery(
                     field2,
-                    new BytesRef(randomFrom(stringContent.get(field1))),
-                    new BytesRef(randomFrom(stringContent.get(field1)))
+                    List.of(new BytesRef(randomFrom(stringContent.get(field1))), new BytesRef(randomFrom(stringContent.get(field1))))
                 )
             );
             // many iterations with boolean queries, which are the most complex queries to deal with when nested
@@ -647,7 +647,7 @@ public void testRangeQueries() throws Exception {
             v
         );
         TopDocs topDocs = shardSearcher.search(query, 1);
-        assertEquals(1L, topDocs.totalHits.value);
+        assertEquals(1L, topDocs.totalHits.value());
         assertEquals(1, topDocs.scoreDocs.length);
         assertEquals(0, topDocs.scoreDocs[0].doc);
 
@@ -655,7 +655,7 @@ public void testRangeQueries() throws Exception {
         percolateSearcher = memoryIndex.createSearcher();
         query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
         topDocs = shardSearcher.search(query, 1);
-        assertEquals(1L, topDocs.totalHits.value);
+        assertEquals(1L, topDocs.totalHits.value());
         assertEquals(1, topDocs.scoreDocs.length);
         assertEquals(1, topDocs.scoreDocs[0].doc);
 
@@ -663,7 +663,7 @@ public void testRangeQueries() throws Exception {
         percolateSearcher = memoryIndex.createSearcher();
         query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
         topDocs = shardSearcher.search(query, 1);
-        assertEquals(1L, topDocs.totalHits.value);
+        assertEquals(1L, topDocs.totalHits.value());
         assertEquals(1, topDocs.scoreDocs.length);
         assertEquals(2, topDocs.scoreDocs[0].doc);
 
@@ -671,7 +671,7 @@ public void testRangeQueries() throws Exception {
         percolateSearcher = memoryIndex.createSearcher();
         query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
         topDocs = shardSearcher.search(query, 1);
-        assertEquals(1, topDocs.totalHits.value);
+        assertEquals(1, topDocs.totalHits.value());
         assertEquals(1, topDocs.scoreDocs.length);
         assertEquals(3, topDocs.scoreDocs[0].doc);
 
@@ -679,7 +679,7 @@ public void testRangeQueries() throws Exception {
         percolateSearcher = memoryIndex.createSearcher();
         query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
         topDocs = shardSearcher.search(query, 1);
-        assertEquals(1, topDocs.totalHits.value);
+        assertEquals(1, topDocs.totalHits.value());
         assertEquals(1, topDocs.scoreDocs.length);
         assertEquals(4, topDocs.scoreDocs[0].doc);
 
@@ -690,7 +690,7 @@ public void testRangeQueries() throws Exception {
         percolateSearcher = memoryIndex.createSearcher();
         query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
         topDocs = shardSearcher.search(query, 1);
-        assertEquals(1, topDocs.totalHits.value);
+        assertEquals(1, topDocs.totalHits.value());
         assertEquals(1, topDocs.scoreDocs.length);
         assertEquals(5, topDocs.scoreDocs[0].doc);
     }
@@ -836,14 +836,14 @@ public void testPercolateMatchAll() throws Exception {
             IndexVersion.current()
         );
         TopDocs topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-        assertEquals(3L, topDocs.totalHits.value);
+        assertEquals(3L, topDocs.totalHits.value());
         assertEquals(3, topDocs.scoreDocs.length);
         assertEquals(0, topDocs.scoreDocs[0].doc);
         assertEquals(1, topDocs.scoreDocs[1].doc);
         assertEquals(4, topDocs.scoreDocs[2].doc);
 
         topDocs = shardSearcher.search(new ConstantScoreQuery(query), 10);
-        assertEquals(3L, topDocs.totalHits.value);
+        assertEquals(3L, topDocs.totalHits.value());
         assertEquals(3, topDocs.scoreDocs.length);
         assertEquals(0, topDocs.scoreDocs[0].doc);
         assertEquals(1, topDocs.scoreDocs[1].doc);
@@ -875,7 +875,7 @@ public void testFunctionScoreQuery() throws Exception {
             IndexVersion.current()
         );
         TopDocs topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-        assertEquals(2L, topDocs.totalHits.value);
+        assertEquals(2L, topDocs.totalHits.value());
         assertEquals(2, topDocs.scoreDocs.length);
         assertEquals(0, topDocs.scoreDocs[0].doc);
         assertEquals(2, topDocs.scoreDocs[1].doc);
@@ -931,15 +931,15 @@ public void testPercolateSmallAndLargeDocument() throws Exception {
                     v
                 );
                 BooleanQuery candidateQuery = (BooleanQuery) query.getCandidateMatchesQuery();
-                assertThat(candidateQuery.clauses().get(0).getQuery(), instanceOf(CoveringQuery.class));
+                assertThat(candidateQuery.clauses().get(0).query(), instanceOf(CoveringQuery.class));
                 TopDocs topDocs = shardSearcher.search(query, 10);
-                assertEquals(2L, topDocs.totalHits.value);
+                assertEquals(2L, topDocs.totalHits.value());
                 assertEquals(2, topDocs.scoreDocs.length);
                 assertEquals(0, topDocs.scoreDocs[0].doc);
                 assertEquals(2, topDocs.scoreDocs[1].doc);
 
                 topDocs = shardSearcher.search(new ConstantScoreQuery(query), 10);
-                assertEquals(2L, topDocs.totalHits.value);
+                assertEquals(2L, topDocs.totalHits.value());
                 assertEquals(2, topDocs.scoreDocs.length);
                 assertEquals(0, topDocs.scoreDocs[0].doc);
                 assertEquals(2, topDocs.scoreDocs[1].doc);
@@ -947,10 +947,10 @@ public void testPercolateSmallAndLargeDocument() throws Exception {
         }
 
         // This will trigger using the TermsQuery instead of individual term query clauses in the CoveringQuery:
-        int origMaxClauseCount = BooleanQuery.getMaxClauseCount();
+        int origMaxClauseCount = IndexSearcher.getMaxClauseCount();
         try (Directory directory = new ByteBuffersDirectory()) {
             final int maxClauseCount = 100;
-            BooleanQuery.setMaxClauseCount(maxClauseCount);
+            IndexSearcher.setMaxClauseCount(maxClauseCount);
             try (IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig())) {
                 Document document = new Document();
                 for (int i = 0; i < maxClauseCount; i++) {
@@ -970,22 +970,22 @@ public void testPercolateSmallAndLargeDocument() throws Exception {
                     v
                 );
                 BooleanQuery candidateQuery = (BooleanQuery) query.getCandidateMatchesQuery();
-                assertThat(candidateQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
+                assertThat(candidateQuery.clauses().get(0).query(), instanceOf(TermInSetQuery.class));
 
                 TopDocs topDocs = shardSearcher.search(query, 10);
-                assertEquals(2L, topDocs.totalHits.value);
+                assertEquals(2L, topDocs.totalHits.value());
                 assertEquals(2, topDocs.scoreDocs.length);
                 assertEquals(1, topDocs.scoreDocs[0].doc);
                 assertEquals(2, topDocs.scoreDocs[1].doc);
 
                 topDocs = shardSearcher.search(new ConstantScoreQuery(query), 10);
-                assertEquals(2L, topDocs.totalHits.value);
+                assertEquals(2L, topDocs.totalHits.value());
                 assertEquals(2, topDocs.scoreDocs.length);
                 assertEquals(1, topDocs.scoreDocs[0].doc);
                 assertEquals(2, topDocs.scoreDocs[1].doc);
             }
         } finally {
-            BooleanQuery.setMaxClauseCount(origMaxClauseCount);
+            IndexSearcher.setMaxClauseCount(origMaxClauseCount);
         }
     }
 
@@ -1032,7 +1032,7 @@ public void testDuplicatedClauses() throws Exception {
         IndexSearcher percolateSearcher = memoryIndex.createSearcher();
         PercolateQuery query = (PercolateQuery) fieldType.percolateQuery("_name", queryStore, sources, percolateSearcher, false, v);
         TopDocs topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-        assertEquals(2L, topDocs.totalHits.value);
+        assertEquals(2L, topDocs.totalHits.value());
         assertEquals(0, topDocs.scoreDocs[0].doc);
         assertEquals(1, topDocs.scoreDocs[1].doc);
     }
@@ -1066,7 +1066,7 @@ public void testDuplicatedClauses2() throws Exception {
         IndexSearcher percolateSearcher = memoryIndex.createSearcher();
         PercolateQuery query = (PercolateQuery) fieldType.percolateQuery("_name", queryStore, sources, percolateSearcher, false, v);
         TopDocs topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-        assertEquals(1L, topDocs.totalHits.value);
+        assertEquals(1L, topDocs.totalHits.value());
         assertEquals(0, topDocs.scoreDocs[0].doc);
 
         memoryIndex = new MemoryIndex();
@@ -1074,7 +1074,7 @@ public void testDuplicatedClauses2() throws Exception {
         percolateSearcher = memoryIndex.createSearcher();
         query = (PercolateQuery) fieldType.percolateQuery("_name", queryStore, sources, percolateSearcher, false, v);
         topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-        assertEquals(1L, topDocs.totalHits.value);
+        assertEquals(1L, topDocs.totalHits.value());
         assertEquals(0, topDocs.scoreDocs[0].doc);
 
         memoryIndex = new MemoryIndex();
@@ -1082,7 +1082,7 @@ public void testDuplicatedClauses2() throws Exception {
         percolateSearcher = memoryIndex.createSearcher();
         query = (PercolateQuery) fieldType.percolateQuery("_name", queryStore, sources, percolateSearcher, false, v);
         topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-        assertEquals(1L, topDocs.totalHits.value);
+        assertEquals(1L, topDocs.totalHits.value());
         assertEquals(0, topDocs.scoreDocs[0].doc);
     }
 
@@ -1117,7 +1117,7 @@ public void testMsmAndRanges_disjunction() throws Exception {
         IndexSearcher percolateSearcher = memoryIndex.createSearcher();
         PercolateQuery query = (PercolateQuery) fieldType.percolateQuery("_name", queryStore, sources, percolateSearcher, false, v);
         TopDocs topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-        assertEquals(1L, topDocs.totalHits.value);
+        assertEquals(1L, topDocs.totalHits.value());
         assertEquals(0, topDocs.scoreDocs[0].doc);
     }
 
@@ -1141,7 +1141,7 @@ private void duelRun(PercolateQuery.QueryStore percolateQueryStore, MemoryIndex
         TopDocs controlTopDocs = shardSearcher.search(controlQuery, 100);
 
         try {
-            assertThat(topDocs.totalHits.value, equalTo(controlTopDocs.totalHits.value));
+            assertThat(topDocs.totalHits.value(), equalTo(controlTopDocs.totalHits.value()));
             assertThat(topDocs.scoreDocs.length, equalTo(controlTopDocs.scoreDocs.length));
             for (int j = 0; j < topDocs.scoreDocs.length; j++) {
                 assertThat(topDocs.scoreDocs[j].doc, equalTo(controlTopDocs.scoreDocs[j].doc));
@@ -1164,12 +1164,13 @@ private void duelRun(PercolateQuery.QueryStore percolateQueryStore, MemoryIndex
                 logger.error("topDocs.scoreDocs[{}].doc={}", i, topDocs.scoreDocs[i].doc);
                 logger.error("topDocs.scoreDocs[{}].score={}", i, topDocs.scoreDocs[i].score);
             }
+            StoredFields storedFields = shardSearcher.storedFields();
             for (int i = 0; i < controlTopDocs.scoreDocs.length; i++) {
                 logger.error("controlTopDocs.scoreDocs[{}].doc={}", i, controlTopDocs.scoreDocs[i].doc);
                 logger.error("controlTopDocs.scoreDocs[{}].score={}", i, controlTopDocs.scoreDocs[i].score);
 
                 // Additional stored information that is useful when debugging:
-                String queryToString = shardSearcher.doc(controlTopDocs.scoreDocs[i].doc).get("query_to_string");
+                String queryToString = storedFields.document(controlTopDocs.scoreDocs[i].doc).get("query_to_string");
                 logger.error("controlTopDocs.scoreDocs[{}].query_to_string={}", i, queryToString);
 
                 TermsEnum tenum = MultiTerms.getTerms(shardSearcher.getIndexReader(), fieldType.queryTermsField.name()).iterator();
@@ -1289,7 +1290,7 @@ public String toString() {
                 }
 
                 @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
+                public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                     float _score[] = new float[] { boost };
                     DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc());
                     CheckedFunction<Integer, Query, IOException> leaf = queryStore.getQueries(context);
@@ -1313,7 +1314,7 @@ protected boolean match(int doc) {
                             }
                         }
                     };
-                    return new Scorer(this) {
+                    Scorer scorer = new Scorer() {
 
                         @Override
                         public int docID() {
@@ -1335,6 +1336,7 @@ public float getMaxScore(int upTo) throws IOException {
                             return _score[0];
                         }
                     };
+                    return new DefaultScorerSupplier(scorer);
                 }
 
                 @Override
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
index 075d4d429fb39..04a8105b5fb82 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
@@ -118,7 +118,7 @@ public void testPercolateQuery() throws Exception {
             )
         );
         TopDocs topDocs = shardSearcher.search(query, 10);
-        assertThat(topDocs.totalHits.value, equalTo(1L));
+        assertThat(topDocs.totalHits.value(), equalTo(1L));
         assertThat(topDocs.scoreDocs.length, equalTo(1));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
         Explanation explanation = shardSearcher.explain(query, 0);
@@ -137,7 +137,7 @@ public void testPercolateQuery() throws Exception {
             )
         );
         topDocs = shardSearcher.search(query, 10);
-        assertThat(topDocs.totalHits.value, equalTo(3L));
+        assertThat(topDocs.totalHits.value(), equalTo(3L));
         assertThat(topDocs.scoreDocs.length, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         explanation = shardSearcher.explain(query, 1);
@@ -166,7 +166,7 @@ public void testPercolateQuery() throws Exception {
             )
         );
         topDocs = shardSearcher.search(query, 10);
-        assertThat(topDocs.totalHits.value, equalTo(4L));
+        assertThat(topDocs.totalHits.value(), equalTo(4L));
 
         query = new PercolateQuery(
             "_name",
@@ -178,7 +178,7 @@ public void testPercolateQuery() throws Exception {
             new MatchNoDocsQuery("")
         );
         topDocs = shardSearcher.search(query, 10);
-        assertThat(topDocs.totalHits.value, equalTo(3L));
+        assertThat(topDocs.totalHits.value(), equalTo(3L));
         assertThat(topDocs.scoreDocs.length, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(3));
         explanation = shardSearcher.explain(query, 3);
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index 100cda66acdcc..f72c68c6fd2e3 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.sandbox.search.CoveringQuery;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
@@ -417,10 +418,10 @@ public void testExtractTermsAndRanges() throws Exception {
     }
 
     public void testCreateCandidateQuery() throws Exception {
-        int origMaxClauseCount = BooleanQuery.getMaxClauseCount();
+        int origMaxClauseCount = IndexSearcher.getMaxClauseCount();
         try {
             final int maxClauseCount = 100;
-            BooleanQuery.setMaxClauseCount(maxClauseCount);
+            IndexSearcher.setMaxClauseCount(maxClauseCount);
             addQueryFieldMappings();
 
             MemoryIndex memoryIndex = new MemoryIndex(false);
@@ -435,8 +436,8 @@ public void testCreateCandidateQuery() throws Exception {
             Tuple<BooleanQuery, Boolean> t = fieldType.createCandidateQuery(indexReader);
             assertTrue(t.v2());
             assertEquals(2, t.v1().clauses().size());
-            assertThat(t.v1().clauses().get(0).getQuery(), instanceOf(CoveringQuery.class));
-            assertThat(t.v1().clauses().get(1).getQuery(), instanceOf(TermQuery.class));
+            assertThat(t.v1().clauses().get(0).query(), instanceOf(CoveringQuery.class));
+            assertThat(t.v1().clauses().get(1).query(), instanceOf(TermQuery.class));
 
             // Now push it over the edge, so that it falls back using TermInSetQuery
             memoryIndex.addField("field2", "value", new WhitespaceAnalyzer());
@@ -444,12 +445,12 @@ public void testCreateCandidateQuery() throws Exception {
             t = fieldType.createCandidateQuery(indexReader);
             assertFalse(t.v2());
             assertEquals(3, t.v1().clauses().size());
-            TermInSetQuery terms = (TermInSetQuery) t.v1().clauses().get(0).getQuery();
-            assertEquals(maxClauseCount - 1, terms.getTermData().size());
-            assertThat(t.v1().clauses().get(1).getQuery().toString(), containsString(fieldName + ".range_field:<ranges:"));
-            assertThat(t.v1().clauses().get(2).getQuery().toString(), containsString(fieldName + ".extraction_result:failed"));
+            TermInSetQuery terms = (TermInSetQuery) t.v1().clauses().get(0).query();
+            assertEquals(maxClauseCount - 1, terms.getTermsCount());
+            assertThat(t.v1().clauses().get(1).query().toString(), containsString(fieldName + ".range_field:<ranges:"));
+            assertThat(t.v1().clauses().get(2).query().toString(), containsString(fieldName + ".extraction_result:failed"));
         } finally {
-            BooleanQuery.setMaxClauseCount(origMaxClauseCount);
+            IndexSearcher.setMaxClauseCount(origMaxClauseCount);
         }
     }
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java
index d9ea30dbd1b68..e84153fababce 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java
@@ -409,7 +409,7 @@ public void testPercolateNamedQueries() {
 
         QueryBuilder query = new PercolateQueryBuilder("my_query", List.of(house1_doc, house2_doc), XContentType.JSON);
         assertResponse(client().prepareSearch("houses").setQuery(query), response -> {
-            assertEquals(2, response.getHits().getTotalHits().value);
+            assertEquals(2, response.getHits().getTotalHits().value());
 
             SearchHit[] hits = response.getHits().getHits();
             assertThat(hits[0].getFields().get("_percolator_document_slot").getValues(), equalTo(Arrays.asList(0, 1)));
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
index a9c3e09e7f4ed..81427060615ea 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
@@ -82,7 +82,7 @@ public void testExtractQueryMetadata_termQuery() {
     }
 
     public void testExtractQueryMetadata_termsQuery() {
-        TermInSetQuery termsQuery = new TermInSetQuery("_field", new BytesRef("_term1"), new BytesRef("_term2"));
+        TermInSetQuery termsQuery = new TermInSetQuery("_field", List.of(new BytesRef("_term1"), new BytesRef("_term2")));
         Result result = analyze(termsQuery);
         assertThat(result.verified, is(true));
         assertThat(result.minimumShouldMatch, equalTo(1));
diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java
index a76ddf13e4595..8b94337141243 100644
--- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java
+++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java
@@ -70,7 +70,7 @@ public void testReindexFromRemoteGivenIndexExists() throws Exception {
             final TotalHits totalHits = SearchResponseUtils.getTotalHits(
                 client(LOCAL_CLUSTER).prepareSearch("desc-index-001").setQuery(new MatchAllQueryBuilder()).setSize(1000)
             );
-            return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber;
+            return totalHits.relation() == TotalHits.Relation.EQUAL_TO && totalHits.value() == docsNumber;
         }));
     }
 
@@ -85,7 +85,7 @@ public void testReindexFromRemoteGivenSameIndexNames() throws Exception {
             final TotalHits totalHits = SearchResponseUtils.getTotalHits(
                 client(LOCAL_CLUSTER).prepareSearch("test-index-001").setQuery(new MatchAllQueryBuilder()).setSize(1000)
             );
-            return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber;
+            return totalHits.relation() == TotalHits.Relation.EQUAL_TO && totalHits.value() == docsNumber;
         }));
     }
 
@@ -114,7 +114,7 @@ public void testReindexManyTimesFromRemoteGivenSameIndexNames() throws Exception
                 final TotalHits totalHits = SearchResponseUtils.getTotalHits(
                     client(LOCAL_CLUSTER).prepareSearch("test-index-001").setQuery(new MatchAllQueryBuilder()).setSize(1000)
                 );
-                return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber;
+                return totalHits.relation() == TotalHits.Relation.EQUAL_TO && totalHits.value() == docsNumber;
             }));
         }
     }
@@ -146,7 +146,7 @@ public void testReindexFromRemoteGivenSimpleDateMathIndexName() throws Interrupt
             final TotalHits totalHits = SearchResponseUtils.getTotalHits(
                 client(LOCAL_CLUSTER).prepareSearch("desc-index-001").setQuery(new MatchAllQueryBuilder()).setSize(1000)
             );
-            return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber;
+            return totalHits.relation() == TotalHits.Relation.EQUAL_TO && totalHits.value() == docsNumber;
         }));
     }
 
@@ -162,7 +162,7 @@ public void testReindexFromRemoteGivenComplexDateMathIndexName() throws Interrup
             final TotalHits totalHits = SearchResponseUtils.getTotalHits(
                 client(LOCAL_CLUSTER).prepareSearch("desc-index-001").setQuery(new MatchAllQueryBuilder()).setSize(1000)
             );
-            return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber;
+            return totalHits.relation() == TotalHits.Relation.EQUAL_TO && totalHits.value() == docsNumber;
         }));
     }
 
diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexNodeShutdownIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexNodeShutdownIT.java
new file mode 100644
index 0000000000000..4a001bb2d0969
--- /dev/null
+++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexNodeShutdownIT.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.node.ShutdownPrepareService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.node.ShutdownPrepareService.MAXIMUM_REINDEXING_TIMEOUT_SETTING;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+
+/**
+ * Test that a wait added during shutdown is necessary for a large reindexing task to complete.
+ * The test works as follows:
+ * 1. Start a large (reasonably long running) reindexing request on the coordinator-only node.
+ * 2. Check that the reindexing task appears on the coordinating node
+ * 3. With a 10s timeout value for MAXIMUM_REINDEXING_TIMEOUT_SETTING,
+ *    wait for the reindexing task to complete before closing the node
+ * 4. Confirm that the reindexing task succeeds with the wait (it will fail without it)
+ */
+@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST)
+public class ReindexNodeShutdownIT extends ESIntegTestCase {
+
+    protected static final String INDEX = "reindex-shutdown-index";
+    protected static final String DEST_INDEX = "dest-index";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(ReindexPlugin.class);
+    }
+
+    protected ReindexRequestBuilder reindex(String nodeName) {
+        return new ReindexRequestBuilder(internalCluster().client(nodeName));
+    }
+
+    public void testReindexWithShutdown() throws Exception {
+        final String masterNodeName = internalCluster().startMasterOnlyNode();
+        final String dataNodeName = internalCluster().startDataOnlyNode();
+
+        final Settings COORD_SETTINGS = Settings.builder()
+            .put(MAXIMUM_REINDEXING_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(10))
+            .build();
+        final String coordNodeName = internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
+
+        ensureStableCluster(3);
+
+        int numDocs = 20000;
+        createIndex(numDocs);
+        createReindexTaskAndShutdown(coordNodeName);
+        checkDestinationIndex(dataNodeName, numDocs);
+    }
+
+    private void createIndex(int numDocs) {
+        // INDEX will be created on the dataNode
+        createIndex(INDEX);
+
+        logger.debug("setting up [{}] docs", numDocs);
+        indexRandom(
+            true,
+            false,
+            true,
+            IntStream.range(0, numDocs)
+                .mapToObj(i -> prepareIndex(INDEX).setId(String.valueOf(i)).setSource("n", i))
+                .collect(Collectors.toList())
+        );
+
+        // Checks that the all documents have been indexed and correctly counted
+        assertHitCount(prepareSearch(INDEX).setSize(0).setTrackTotalHits(true), numDocs);
+    }
+
+    private void createReindexTaskAndShutdown(final String coordNodeName) throws Exception {
+        AbstractBulkByScrollRequestBuilder<?, ?> builder = reindex(coordNodeName).source(INDEX).destination(DEST_INDEX);
+        AbstractBulkByScrollRequest<?> reindexRequest = builder.request();
+        ShutdownPrepareService shutdownPrepareService = internalCluster().getInstance(ShutdownPrepareService.class, coordNodeName);
+
+        TaskManager taskManager = internalCluster().getInstance(TransportService.class, coordNodeName).getTaskManager();
+
+        // Now execute the reindex action...
+        ActionListener<BulkByScrollResponse> reindexListener = new ActionListener<BulkByScrollResponse>() {
+            @Override
+            public void onResponse(BulkByScrollResponse bulkByScrollResponse) {
+                assertNull(bulkByScrollResponse.getReasonCancelled());
+                logger.debug(bulkByScrollResponse.toString());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.debug("Encounterd " + e.toString());
+                fail(e, "Encounterd " + e.toString());
+            }
+        };
+        internalCluster().client(coordNodeName).execute(ReindexAction.INSTANCE, reindexRequest, reindexListener);
+
+        // Check for reindex task to appear in the tasks list and Immediately stop coordinating node
+        waitForTask(ReindexAction.INSTANCE.name(), coordNodeName);
+        shutdownPrepareService.prepareForShutdown(taskManager);
+        internalCluster().stopNode(coordNodeName);
+    }
+
+    // Make sure all documents from the source index have been reindexed into the destination index
+    private void checkDestinationIndex(String dataNodeName, int numDocs) throws Exception {
+        assertTrue(indexExists(DEST_INDEX));
+        flushAndRefresh(DEST_INDEX);
+        assertBusy(() -> { assertHitCount(prepareSearch(DEST_INDEX).setSize(0).setTrackTotalHits(true), numDocs); });
+    }
+
+    private static void waitForTask(String actionName, String nodeName) throws Exception {
+        assertBusy(() -> {
+            ListTasksResponse tasks = clusterAdmin().prepareListTasks(nodeName).setActions(actionName).setDetailed(true).get();
+            tasks.rethrowFailures("Find my task");
+            for (TaskInfo taskInfo : tasks.getTasks()) {
+                // Skip tasks with a parent because those are children of the task we want
+                if (taskInfo.parentTaskId().isSet() == false) return;
+            }
+            fail("Couldn't find task after waiting, tasks=" + tasks.getTasks());
+        }, 10, TimeUnit.SECONDS);
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java
index 48a892f796f92..095d119bf2719 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -28,7 +27,6 @@
 import java.io.IOException;
 import java.util.Map;
 import java.util.function.Consumer;
-import java.util.function.IntConsumer;
 import java.util.function.Predicate;
 
 /**
@@ -54,10 +52,7 @@ protected void parseInternalRequest(
         SearchRequest searchRequest = internal.getSearchRequest();
 
         try (XContentParser parser = extractRequestSpecificFields(restRequest, bodyConsumers)) {
-            IntConsumer sizeConsumer = restRequest.getRestApiVersion() == RestApiVersion.V_7
-                ? size -> setMaxDocsFromSearchSize(internal, size)
-                : size -> failOnSizeSpecified();
-            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, clusterSupportsFeature, sizeConsumer);
+            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, clusterSupportsFeature, size -> failOnSizeSpecified());
         }
 
         searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size()));
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java
index 4b960e97ce0e0..d046ba881b5d4 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocWriteRequest;
@@ -96,7 +95,7 @@ static CharacterRunAutomaton buildRemoteWhitelist(List<String> whitelist) {
             return new CharacterRunAutomaton(Automata.makeEmpty());
         }
         Automaton automaton = Regex.simpleMatchToAutomaton(whitelist.toArray(Strings.EMPTY_ARRAY));
-        automaton = MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        automaton = Operations.determinize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
         if (Operations.isTotal(automaton)) {
             throw new IllegalArgumentException(
                 "Refusing to start because whitelist "
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java
index 920968cc8cbca..29c1519afa4d1 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java
@@ -10,14 +10,12 @@
 package org.elasticsearch.reindex;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
-import org.elasticsearch.rest.action.search.RestSearchAction;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -40,13 +38,7 @@ public RestDeleteByQueryAction(Predicate<NodeFeature> clusterSupportsFeature) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(POST, "/{index}/_delete_by_query"),
-            Route.builder(POST, "/{index}/{type}/_delete_by_query")
-                .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7)
-                .build()
-        );
-
+        return List.of(new Route(POST, "/{index}/_delete_by_query"));
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java
index 32eb34cbf4c71..632ed73b9b2fa 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java
@@ -10,14 +10,12 @@
 package org.elasticsearch.reindex;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.reindex.UpdateByQueryAction;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
-import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.Script;
 
 import java.io.IOException;
@@ -41,12 +39,7 @@ public RestUpdateByQueryAction(Predicate<NodeFeature> clusterSupportsFeature) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(POST, "/{index}/_update_by_query"),
-            Route.builder(POST, "/{index}/{type}/_update_by_query")
-                .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, "/{index}/_update_by_query"));
     }
 
     @Override
@@ -61,9 +54,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
 
     @Override
     protected UpdateByQueryRequest buildRequest(RestRequest request) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type");
-        }
         /*
          * Passing the search request through UpdateByQueryRequest first allows
          * it to set its own defaults which differ from SearchRequest's
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java
index b924f8c311115..01459e2ff61bb 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java
@@ -97,8 +97,8 @@ class Fields {
         HITS_PARSER.declareField(constructorArg(), (p, c) -> {
             if (p.currentToken() == XContentParser.Token.START_OBJECT) {
                 final TotalHits totalHits = SearchHits.parseTotalHitsFragment(p);
-                assert totalHits.relation == TotalHits.Relation.EQUAL_TO;
-                return totalHits.value;
+                assert totalHits.relation() == TotalHits.Relation.EQUAL_TO;
+                return totalHits.value();
             } else {
                 // For BWC with nodes pre 7.0
                 return p.longValue();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java
index a6e28477f8582..ebb4471566fbd 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 
-import static java.util.Collections.emptyList;
 import static org.elasticsearch.reindex.BulkByScrollParallelizationHelper.sliceIntoSubRequests;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder;
@@ -24,7 +24,7 @@
 public class BulkByScrollParallelizationHelperTests extends ESTestCase {
     public void testSliceIntoSubRequests() throws IOException {
         SearchRequest searchRequest = randomSearchRequest(
-            () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, () -> null, () -> emptyList(), () -> null, () -> null)
+            () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, Collections::emptyList, () -> null, () -> null)
         );
         if (searchRequest.source() != null) {
             // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is.
diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle
index eb938f663c810..d011de81f4fb3 100644
--- a/modules/repository-azure/build.gradle
+++ b/modules/repository-azure/build.gradle
@@ -30,6 +30,7 @@ dependencies {
   api "com.azure:azure-identity:1.13.2"
   api "com.azure:azure-json:1.2.0"
   api "com.azure:azure-storage-blob:12.27.1"
+  api "com.azure:azure-storage-blob-batch:12.23.1"
   api "com.azure:azure-storage-common:12.26.1"
   api "com.azure:azure-storage-internal-avro:12.12.1"
   api "com.azure:azure-xml:1.1.0"
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java
index a9bf0afa37e18..61940be247861 100644
--- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java
@@ -9,14 +9,18 @@
 
 package org.elasticsearch.repositories.azure;
 
+import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.OperationPurpose;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.RepositoriesMetrics;
@@ -31,6 +35,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Queue;
@@ -43,6 +48,7 @@
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.repositories.azure.AbstractAzureServerTestCase.randomBlobContent;
+import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -225,6 +231,91 @@ public void testRequestTimeIsAccurate() throws IOException {
         assertThat(recordedRequestTime, lessThanOrEqualTo(elapsedTimeMillis));
     }
 
+    public void testBatchDeleteFailure() throws IOException {
+        final int deleteBatchSize = randomIntBetween(1, 30);
+        final String repositoryName = randomRepositoryName();
+        final String repository = createRepository(
+            repositoryName,
+            Settings.builder()
+                .put(repositorySettings(repositoryName))
+                .put(AzureRepository.Repository.DELETION_BATCH_SIZE_SETTING.getKey(), deleteBatchSize)
+                .build(),
+            true
+        );
+        final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData);
+        final BlobContainer container = getBlobContainer(dataNodeName, repository);
+
+        final List<String> blobsToDelete = new ArrayList<>();
+        final int numberOfBatches = randomIntBetween(3, 20);
+        final int numberOfBlobs = numberOfBatches * deleteBatchSize;
+        final int failedBatches = randomIntBetween(1, numberOfBatches);
+        for (int i = 0; i < numberOfBlobs; i++) {
+            byte[] bytes = randomBytes(randomInt(100));
+            String blobName = "index-" + randomAlphaOfLength(10);
+            container.writeBlob(randomPurpose(), blobName, new BytesArray(bytes), false);
+            blobsToDelete.add(blobName);
+        }
+        Randomness.shuffle(blobsToDelete);
+        clearMetrics(dataNodeName);
+
+        // Handler will fail one or more of the batch requests
+        final RequestHandler failNRequestRequestHandler = createFailNRequestsHandler(failedBatches);
+
+        // Exhaust the retries
+        IntStream.range(0, (numberOfBatches - failedBatches) + (failedBatches * (MAX_RETRIES + 1)))
+            .forEach(i -> requestHandlers.offer(failNRequestRequestHandler));
+
+        logger.info("--> Failing {} of {} batches", failedBatches, numberOfBatches);
+
+        final IOException exception = assertThrows(
+            IOException.class,
+            () -> container.deleteBlobsIgnoringIfNotExists(randomPurpose(), blobsToDelete.iterator())
+        );
+        assertEquals(Math.min(failedBatches, 10), exception.getSuppressed().length);
+        assertEquals(
+            (numberOfBatches - failedBatches) + (failedBatches * (MAX_RETRIES + 1L)),
+            getLongCounterTotal(dataNodeName, RepositoriesMetrics.METRIC_REQUESTS_TOTAL)
+        );
+        assertEquals((failedBatches * (MAX_RETRIES + 1L)), getLongCounterTotal(dataNodeName, RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL));
+        assertEquals(failedBatches * deleteBatchSize, container.listBlobs(randomPurpose()).size());
+    }
+
+    private long getLongCounterTotal(String dataNodeName, String metricKey) {
+        return getTelemetryPlugin(dataNodeName).getLongCounterMeasurement(metricKey)
+            .stream()
+            .mapToLong(Measurement::getLong)
+            .reduce(0L, Long::sum);
+    }
+
+    /**
+     * Creates a {@link RequestHandler} that will persistently fail the first <code>numberToFail</code> distinct requests
+     * it sees. Any other requests are passed through to the delegate.
+     *
+     * @param numberToFail The number of requests to fail
+     * @return the handler
+     */
+    private static RequestHandler createFailNRequestsHandler(int numberToFail) {
+        final List<String> requestsToFail = new ArrayList<>(numberToFail);
+        return (exchange, delegate) -> {
+            final Headers requestHeaders = exchange.getRequestHeaders();
+            final String requestId = requestHeaders.get("X-ms-client-request-id").get(0);
+            boolean failRequest = false;
+            synchronized (requestsToFail) {
+                if (requestsToFail.contains(requestId)) {
+                    failRequest = true;
+                } else if (requestsToFail.size() < numberToFail) {
+                    requestsToFail.add(requestId);
+                    failRequest = true;
+                }
+            }
+            if (failRequest) {
+                exchange.sendResponseHeaders(500, -1);
+            } else {
+                delegate.handle(exchange);
+            }
+        };
+    }
+
     private void clearMetrics(String discoveryNode) {
         internalCluster().getInstance(PluginsService.class, discoveryNode)
             .filterPlugins(TestTelemetryPlugin.class)
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
index 473d91da6e34c..bd21f208faac4 100644
--- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
@@ -89,7 +89,9 @@ protected Settings repositorySettings(String repoName) {
             .put(super.repositorySettings(repoName))
             .put(AzureRepository.Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.MB))
             .put(AzureRepository.Repository.CONTAINER_SETTING.getKey(), "container")
-            .put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test");
+            .put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test")
+            .put(AzureRepository.Repository.DELETION_BATCH_SIZE_SETTING.getKey(), randomIntBetween(5, 256))
+            .put(AzureRepository.Repository.MAX_CONCURRENT_BATCH_DELETES_SETTING.getKey(), randomIntBetween(1, 10));
         if (randomBoolean()) {
             settingsBuilder.put(AzureRepository.Repository.BASE_PATH_SETTING.getKey(), randomFrom("test", "test/1"));
         }
@@ -249,6 +251,8 @@ protected void maybeTrack(String request, Headers headers) {
                 trackRequest("PutBlockList");
             } else if (Regex.simpleMatch("PUT /*/*", request)) {
                 trackRequest("PutBlob");
+            } else if (Regex.simpleMatch("POST /*/*?*comp=batch*", request)) {
+                trackRequest("BlobBatch");
             }
         }
 
@@ -279,10 +283,22 @@ public void testLargeBlobCountDeletion() throws Exception {
     }
 
     public void testDeleteBlobsIgnoringIfNotExists() throws Exception {
-        try (BlobStore store = newBlobStore()) {
+        // Test with a smaller batch size here
+        final int deleteBatchSize = randomIntBetween(1, 30);
+        final String repositoryName = randomRepositoryName();
+        createRepository(
+            repositoryName,
+            Settings.builder()
+                .put(repositorySettings(repositoryName))
+                .put(AzureRepository.Repository.DELETION_BATCH_SIZE_SETTING.getKey(), deleteBatchSize)
+                .build(),
+            true
+        );
+        try (BlobStore store = newBlobStore(repositoryName)) {
             final BlobContainer container = store.blobContainer(BlobPath.EMPTY);
-            List<String> blobsToDelete = new ArrayList<>();
-            for (int i = 0; i < 10; i++) {
+            final int toDeleteCount = randomIntBetween(deleteBatchSize, 3 * deleteBatchSize);
+            final List<String> blobsToDelete = new ArrayList<>();
+            for (int i = 0; i < toDeleteCount; i++) {
                 byte[] bytes = randomBytes(randomInt(100));
                 String blobName = randomAlphaOfLength(10);
                 container.writeBlob(randomPurpose(), blobName, new BytesArray(bytes), false);
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
index abd4f506a0bb3..6d5c17c392141 100644
--- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
@@ -30,6 +30,8 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -46,6 +48,7 @@
 import static org.hamcrest.Matchers.not;
 
 public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+    private static final Logger logger = LogManager.getLogger(AzureStorageCleanupThirdPartyTests.class);
     private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
 
     private static final String AZURE_ACCOUNT = System.getProperty("test.azure.account");
@@ -89,8 +92,10 @@ protected SecureSettings credentials() {
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
         if (hasSasToken) {
+            logger.info("--> Using SAS token authentication");
             secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token"));
         } else {
+            logger.info("--> Using key authentication");
             secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
         }
         return secureSettings;
diff --git a/modules/repository-azure/src/main/java/module-info.java b/modules/repository-azure/src/main/java/module-info.java
index cd6be56b71543..731f1e0a9986a 100644
--- a/modules/repository-azure/src/main/java/module-info.java
+++ b/modules/repository-azure/src/main/java/module-info.java
@@ -18,10 +18,7 @@
     requires org.apache.logging.log4j;
     requires org.apache.logging.log4j.core;
 
-    requires com.azure.core;
     requires com.azure.http.netty;
-    requires com.azure.storage.blob;
-    requires com.azure.storage.common;
     requires com.azure.identity;
 
     requires io.netty.buffer;
@@ -29,7 +26,7 @@
     requires io.netty.resolver;
     requires io.netty.common;
 
-    requires reactor.core;
     requires reactor.netty.core;
     requires reactor.netty.http;
+    requires com.azure.storage.blob.batch;
 }
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
index a3f26424324fa..52bc1ee1399d4 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
@@ -138,7 +138,7 @@ public void writeMetadataBlob(
     }
 
     @Override
-    public DeleteResult delete(OperationPurpose purpose) {
+    public DeleteResult delete(OperationPurpose purpose) throws IOException {
         return blobStore.deleteBlobDirectory(purpose, keyPath);
     }
 
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
index 829868797e38c..3c64bb9f3b830 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
@@ -25,6 +25,10 @@
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobServiceAsyncClient;
 import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.batch.BlobBatch;
+import com.azure.storage.blob.batch.BlobBatchAsyncClient;
+import com.azure.storage.blob.batch.BlobBatchClientBuilder;
+import com.azure.storage.blob.batch.BlobBatchStorageException;
 import com.azure.storage.blob.models.BlobErrorCode;
 import com.azure.storage.blob.models.BlobItem;
 import com.azure.storage.blob.models.BlobItemProperties;
@@ -99,6 +103,8 @@
 
 public class AzureBlobStore implements BlobStore {
     private static final Logger logger = LogManager.getLogger(AzureBlobStore.class);
+    // See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch#request-body
+    public static final int MAX_ELEMENTS_PER_BATCH = 256;
     private static final long DEFAULT_READ_CHUNK_SIZE = new ByteSizeValue(32, ByteSizeUnit.MB).getBytes();
     private static final int DEFAULT_UPLOAD_BUFFERS_SIZE = (int) new ByteSizeValue(64, ByteSizeUnit.KB).getBytes();
 
@@ -110,6 +116,8 @@ public class AzureBlobStore implements BlobStore {
     private final String container;
     private final LocationMode locationMode;
     private final ByteSizeValue maxSinglePartUploadSize;
+    private final int deletionBatchSize;
+    private final int maxConcurrentBatchDeletes;
 
     private final RequestMetricsRecorder requestMetricsRecorder;
     private final AzureClientProvider.RequestMetricsHandler requestMetricsHandler;
@@ -129,6 +137,8 @@ public AzureBlobStore(
         // locationMode is set per repository, not per client
         this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
         this.maxSinglePartUploadSize = Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.get(metadata.settings());
+        this.deletionBatchSize = Repository.DELETION_BATCH_SIZE_SETTING.get(metadata.settings());
+        this.maxConcurrentBatchDeletes = Repository.MAX_CONCURRENT_BATCH_DELETES_SETTING.get(metadata.settings());
 
         List<RequestMatcher> requestMatchers = List.of(
             new RequestMatcher((httpMethod, url) -> httpMethod == HttpMethod.HEAD, Operation.GET_BLOB_PROPERTIES),
@@ -147,17 +157,14 @@ public AzureBlobStore(
                     && isPutBlockRequest(httpMethod, url) == false
                     && isPutBlockListRequest(httpMethod, url) == false,
                 Operation.PUT_BLOB
-            )
+            ),
+            new RequestMatcher(AzureBlobStore::isBlobBatch, Operation.BLOB_BATCH)
         );
 
         this.requestMetricsHandler = (purpose, method, url, metrics) -> {
             try {
                 URI uri = url.toURI();
                 String path = uri.getPath() == null ? "" : uri.getPath();
-                // Batch delete requests
-                if (path.contains(container) == false) {
-                    return;
-                }
                 assert path.contains(container) : uri.toString();
             } catch (URISyntaxException ignored) {
                 return;
@@ -172,6 +179,10 @@ && isPutBlockListRequest(httpMethod, url) == false,
         };
     }
 
+    private static boolean isBlobBatch(HttpMethod method, URL url) {
+        return method == HttpMethod.POST && url.getQuery() != null && url.getQuery().contains("comp=batch");
+    }
+
     private static boolean isListRequest(HttpMethod httpMethod, URL url) {
         return httpMethod == HttpMethod.GET && url.getQuery() != null && url.getQuery().contains("comp=list");
     }
@@ -231,95 +242,101 @@ public boolean blobExists(OperationPurpose purpose, String blob) throws IOExcept
         }
     }
 
-    // number of concurrent blob delete requests to use while bulk deleting
-    private static final int CONCURRENT_DELETES = 100;
-
-    public DeleteResult deleteBlobDirectory(OperationPurpose purpose, String path) {
+    public DeleteResult deleteBlobDirectory(OperationPurpose purpose, String path) throws IOException {
         final AtomicInteger blobsDeleted = new AtomicInteger(0);
         final AtomicLong bytesDeleted = new AtomicLong(0);
 
         SocketAccess.doPrivilegedVoidException(() -> {
-            final BlobContainerAsyncClient blobContainerAsyncClient = asyncClient(purpose).getBlobContainerAsyncClient(container);
+            final AzureBlobServiceClient client = getAzureBlobServiceClientClient(purpose);
+            final BlobContainerAsyncClient blobContainerAsyncClient = client.getAsyncClient().getBlobContainerAsyncClient(container);
             final ListBlobsOptions options = new ListBlobsOptions().setPrefix(path)
                 .setDetails(new BlobListDetails().setRetrieveMetadata(true));
-            try {
-                blobContainerAsyncClient.listBlobs(options, null).flatMap(blobItem -> {
-                    if (blobItem.isPrefix() != null && blobItem.isPrefix()) {
-                        return Mono.empty();
-                    } else {
-                        final String blobName = blobItem.getName();
-                        BlobAsyncClient blobAsyncClient = blobContainerAsyncClient.getBlobAsyncClient(blobName);
-                        final Mono<Void> deleteTask = getDeleteTask(blobName, blobAsyncClient);
-                        bytesDeleted.addAndGet(blobItem.getProperties().getContentLength());
-                        blobsDeleted.incrementAndGet();
-                        return deleteTask;
-                    }
-                }, CONCURRENT_DELETES).then().block();
-            } catch (Exception e) {
-                filterDeleteExceptionsAndRethrow(e, new IOException("Deleting directory [" + path + "] failed"));
-            }
+            final Flux<String> blobsFlux = blobContainerAsyncClient.listBlobs(options).filter(bi -> bi.isPrefix() == false).map(bi -> {
+                bytesDeleted.addAndGet(bi.getProperties().getContentLength());
+                blobsDeleted.incrementAndGet();
+                return bi.getName();
+            });
+            deleteListOfBlobs(client, blobsFlux);
         });
 
         return new DeleteResult(blobsDeleted.get(), bytesDeleted.get());
     }
 
-    private static void filterDeleteExceptionsAndRethrow(Exception e, IOException exception) throws IOException {
-        int suppressedCount = 0;
-        for (Throwable suppressed : e.getSuppressed()) {
-            // We're only interested about the blob deletion exceptions and not in the reactor internals exceptions
-            if (suppressed instanceof IOException) {
-                exception.addSuppressed(suppressed);
-                suppressedCount++;
-                if (suppressedCount > 10) {
-                    break;
-                }
-            }
+    @Override
+    public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<String> blobNames) throws IOException {
+        if (blobNames.hasNext() == false) {
+            return;
+        }
+        SocketAccess.doPrivilegedVoidException(
+            () -> deleteListOfBlobs(
+                getAzureBlobServiceClientClient(purpose),
+                Flux.fromStream(StreamSupport.stream(Spliterators.spliteratorUnknownSize(blobNames, Spliterator.ORDERED), false))
+            )
+        );
+    }
+
+    private void deleteListOfBlobs(AzureBlobServiceClient azureBlobServiceClient, Flux<String> blobNames) throws IOException {
+        // We need to use a container-scoped BlobBatchClient, so the restype=container parameter
+        // is sent, and we can support all SAS token types
+        // See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=shared-access-signatures#authorization
+        final BlobBatchAsyncClient batchAsyncClient = new BlobBatchClientBuilder(
+            azureBlobServiceClient.getAsyncClient().getBlobContainerAsyncClient(container)
+        ).buildAsyncClient();
+        final List<Throwable> errors;
+        final AtomicInteger errorsCollected = new AtomicInteger(0);
+        try {
+            errors = blobNames.buffer(deletionBatchSize).flatMap(blobs -> {
+                final BlobBatch blobBatch = batchAsyncClient.getBlobBatch();
+                blobs.forEach(blob -> blobBatch.deleteBlob(container, blob));
+                return batchAsyncClient.submitBatch(blobBatch).then(Mono.<Throwable>empty()).onErrorResume(t -> {
+                    // Ignore errors that are just 404s, send other errors downstream as values
+                    if (AzureBlobStore.isIgnorableBatchDeleteException(t)) {
+                        return Mono.empty();
+                    } else {
+                        // Propagate the first 10 errors only
+                        if (errorsCollected.getAndIncrement() < 10) {
+                            return Mono.just(t);
+                        } else {
+                            return Mono.empty();
+                        }
+                    }
+                });
+            }, maxConcurrentBatchDeletes).collectList().block();
+        } catch (Exception e) {
+            throw new IOException("Error deleting batches", e);
+        }
+        if (errors.isEmpty() == false) {
+            final int totalErrorCount = errorsCollected.get();
+            final String errorMessage = totalErrorCount > errors.size()
+                ? "Some errors occurred deleting batches, the first "
+                    + errors.size()
+                    + " are included as suppressed, but the total count was "
+                    + totalErrorCount
+                : "Some errors occurred deleting batches, all errors included as suppressed";
+            final IOException ex = new IOException(errorMessage);
+            errors.forEach(ex::addSuppressed);
+            throw ex;
         }
-        throw exception;
     }
 
     /**
-     * {@inheritDoc}
-     * <p>
-     * Note that in this Azure implementation we issue a series of individual
-     * <a href="https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob">delete blob</a> calls rather than aggregating
-     * deletions into <a href="https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch">blob batch</a> calls.
-     * The reason for this is that the blob batch endpoint has limited support for SAS token authentication.
+     * We can ignore {@link BlobBatchStorageException}s when they are just telling us some of the files were not found
      *
-     * @see <a href="https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=shared-access-signatures#authorization">
-     *     API docs around SAS auth limitations</a>
-     * @see <a href="https://github.com/Azure/azure-storage-java/issues/538">Java SDK issue</a>
-     * @see <a href="https://github.com/elastic/elasticsearch/pull/65140#discussion_r528752070">Discussion on implementing PR</a>
+     * @param exception An exception throw by batch delete
+     * @return true if it is safe to ignore, false otherwise
      */
-    @Override
-    public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<String> blobs) {
-        if (blobs.hasNext() == false) {
-            return;
-        }
-
-        BlobServiceAsyncClient asyncClient = asyncClient(purpose);
-        SocketAccess.doPrivilegedVoidException(() -> {
-            final BlobContainerAsyncClient blobContainerClient = asyncClient.getBlobContainerAsyncClient(container);
-            try {
-                Flux.fromStream(StreamSupport.stream(Spliterators.spliteratorUnknownSize(blobs, Spliterator.ORDERED), false))
-                    .flatMap(blob -> getDeleteTask(blob, blobContainerClient.getBlobAsyncClient(blob)), CONCURRENT_DELETES)
-                    .then()
-                    .block();
-            } catch (Exception e) {
-                filterDeleteExceptionsAndRethrow(e, new IOException("Unable to delete blobs"));
+    private static boolean isIgnorableBatchDeleteException(Throwable exception) {
+        if (exception instanceof BlobBatchStorageException bbse) {
+            final Iterable<BlobStorageException> batchExceptions = bbse.getBatchExceptions();
+            for (BlobStorageException bse : batchExceptions) {
+                // If any requests failed with something other than a BLOB_NOT_FOUND, it is not ignorable
+                if (BlobErrorCode.BLOB_NOT_FOUND.equals(bse.getErrorCode()) == false) {
+                    return false;
+                }
             }
-        });
-    }
-
-    private static Mono<Void> getDeleteTask(String blobName, BlobAsyncClient blobAsyncClient) {
-        return blobAsyncClient.delete()
-            // Ignore not found blobs, as it's possible that due to network errors a request
-            // for an already deleted blob is retried, causing an error.
-            .onErrorResume(
-                e -> e instanceof BlobStorageException blobStorageException && blobStorageException.getStatusCode() == 404,
-                throwable -> Mono.empty()
-            )
-            .onErrorMap(throwable -> new IOException("Error deleting blob " + blobName, throwable));
+            return true;
+        }
+        return false;
     }
 
     public InputStream getInputStream(OperationPurpose purpose, String blob, long position, final @Nullable Long length) {
@@ -363,8 +380,7 @@ public Map<String, BlobMetadata> listBlobsByPrefix(OperationPurpose purpose, Str
 
                 for (final BlobItem blobItem : containerClient.listBlobsByHierarchy("/", listBlobsOptions, null)) {
                     BlobItemProperties properties = blobItem.getProperties();
-                    Boolean isPrefix = blobItem.isPrefix();
-                    if (isPrefix != null && isPrefix) {
+                    if (blobItem.isPrefix()) {
                         continue;
                     }
                     String blobName = blobItem.getName().substring(keyPath.length());
@@ -689,7 +705,8 @@ enum Operation {
         GET_BLOB_PROPERTIES("GetBlobProperties"),
         PUT_BLOB("PutBlob"),
         PUT_BLOCK("PutBlock"),
-        PUT_BLOCK_LIST("PutBlockList");
+        PUT_BLOCK_LIST("PutBlockList"),
+        BLOB_BATCH("BlobBatch");
 
         private final String key;
 
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
index 654742c980268..f92bbcbdd716d 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
@@ -317,6 +317,11 @@ private enum RetryMetricsTracker implements HttpPipelinePolicy {
 
         @Override
         public Mono<HttpResponse> process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) {
+            if (requestIsPartOfABatch(context)) {
+                // Batch deletes fire once for each of the constituent requests, and they have a null response. Ignore those, we'll track
+                // metrics at the bulk level.
+                return next.process();
+            }
             Optional<Object> metricsData = context.getData(RequestMetricsTracker.ES_REQUEST_METRICS_CONTEXT_KEY);
             if (metricsData.isPresent() == false) {
                 assert false : "No metrics object associated with request " + context.getHttpRequest();
@@ -361,6 +366,11 @@ private RequestMetricsTracker(OperationPurpose purpose, RequestMetricsHandler re
 
         @Override
         public Mono<HttpResponse> process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) {
+            if (requestIsPartOfABatch(context)) {
+                // Batch deletes fire once for each of the constituent requests, and they have a null response. Ignore those, we'll track
+                // metrics at the bulk level.
+                return next.process();
+            }
             final RequestMetrics requestMetrics = new RequestMetrics();
             context.setData(ES_REQUEST_METRICS_CONTEXT_KEY, requestMetrics);
             return next.process().doOnSuccess((httpResponse) -> {
@@ -389,6 +399,10 @@ public HttpPipelinePosition getPipelinePosition() {
         }
     }
 
+    private static boolean requestIsPartOfABatch(HttpPipelineCallContext context) {
+        return context.getData("Batch-Operation-Info").isPresent();
+    }
+
     /**
      * The {@link RequestMetricsTracker} calls this when a request completes
      */
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 80e662343baee..316db4844e598 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -87,6 +87,21 @@ public static final class Repository {
             DEFAULT_MAX_SINGLE_UPLOAD_SIZE,
             Property.NodeScope
         );
+
+        /**
+         * The batch size for batched delete requests
+         */
+        static final Setting<Integer> DELETION_BATCH_SIZE_SETTING = Setting.intSetting(
+            "delete_objects_max_size",
+            AzureBlobStore.MAX_ELEMENTS_PER_BATCH,
+            1,
+            AzureBlobStore.MAX_ELEMENTS_PER_BATCH
+        );
+
+        /**
+         * The maximum number of concurrent batch deletes
+         */
+        static final Setting<Integer> MAX_CONCURRENT_BATCH_DELETES_SETTING = Setting.intSetting("max_concurrent_batch_deletes", 10, 1, 100);
     }
 
     private final ByteSizeValue chunkSize;
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java
index 1ed01bbadc07e..6730e5c3c81bd 100644
--- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java
@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.List;
 import java.util.Map;
 
 public class AzureBlobContainerStatsTests extends AbstractAzureServerTestCase {
@@ -47,6 +48,8 @@ public void testOperationPurposeIsReflectedInBlobStoreStats() throws IOException
             os.write(blobContent);
             os.flush();
         });
+        // BLOB_BATCH
+        blobStore.deleteBlobsIgnoringIfNotExists(purpose, List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator());
 
         Map<String, Long> stats = blobStore.stats();
         String statsMapString = stats.toString();
@@ -55,6 +58,7 @@ public void testOperationPurposeIsReflectedInBlobStoreStats() throws IOException
         assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES)));
         assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.PUT_BLOCK)));
         assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.PUT_BLOCK_LIST)));
+        assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.BLOB_BATCH)));
     }
 
     public void testOperationPurposeIsNotReflectedInBlobStoreStatsWhenNotServerless() throws IOException {
@@ -79,6 +83,11 @@ public void testOperationPurposeIsNotReflectedInBlobStoreStatsWhenNotServerless(
                 os.write(blobContent);
                 os.flush();
             });
+            // BLOB_BATCH
+            blobStore.deleteBlobsIgnoringIfNotExists(
+                purpose,
+                List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator()
+            );
         }
 
         Map<String, Long> stats = blobStore.stats();
@@ -88,6 +97,7 @@ public void testOperationPurposeIsNotReflectedInBlobStoreStatsWhenNotServerless(
         assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.GET_BLOB_PROPERTIES.getKey()));
         assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.PUT_BLOCK.getKey()));
         assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.PUT_BLOCK_LIST.getKey()));
+        assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.BLOB_BATCH.getKey()));
     }
 
     private static String statsKey(OperationPurpose purpose, AzureBlobStore.Operation operation) {
diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java
index e55668adea101..21f42bf9eb99c 100644
--- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java
+++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -31,6 +32,8 @@
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -38,6 +41,7 @@
 import java.util.Queue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
 
 import static org.elasticsearch.repositories.RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM;
 import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM;
@@ -48,9 +52,11 @@
 import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_THROTTLES_HISTOGRAM;
 import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_THROTTLES_TOTAL;
 import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL;
+import static org.elasticsearch.repositories.s3.S3RepositoriesMetrics.METRIC_DELETE_RETRIES_HISTOGRAM;
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.REQUESTED_RANGE_NOT_SATISFIED;
+import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE;
 import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -61,14 +67,22 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class S3BlobStoreRepositoryMetricsTests extends S3BlobStoreRepositoryTests {
 
-    private final Queue<RestStatus> errorStatusQueue = new LinkedBlockingQueue<>();
+    private static final S3ErrorResponse S3_SLOW_DOWN_RESPONSE = new S3ErrorResponse(SERVICE_UNAVAILABLE, """
+        <?xml version="1.0" encoding="UTF-8"?>
+        <Error>
+          <Code>SlowDown</Code>
+          <Message>This is a throttling message</Message>
+          <Resource>/bucket/</Resource>
+          <RequestId>4442587FB7D0A2F9</RequestId>
+        </Error>""");
+    private final Queue<S3ErrorResponse> errorResponseQueue = new LinkedBlockingQueue<>();
 
     // Always create erroneous handler
     @Override
     protected Map<String, HttpHandler> createHttpHandlers() {
         return Collections.singletonMap(
             "/bucket",
-            new S3StatsCollectorHttpHandler(new S3MetricErroneousHttpHandler(new S3BlobStoreHttpHandler("bucket"), errorStatusQueue))
+            new S3StatsCollectorHttpHandler(new S3MetricErroneousHttpHandler(new S3BlobStoreHttpHandler("bucket"), errorResponseQueue))
         );
     }
 
@@ -244,8 +258,74 @@ public void testMetricsForRequestRangeNotSatisfied() {
         }
     }
 
+    public void testRetrySnapshotDeleteMetricsOnEventualSuccess() throws IOException {
+        final int maxRetries = 5;
+        final String repositoryName = randomRepositoryName();
+        // Disable retries in the client for this repo
+        createRepository(
+            repositoryName,
+            Settings.builder()
+                .put(repositorySettings(repositoryName))
+                .put(S3ClientSettings.MAX_RETRIES_SETTING.getConcreteSettingForNamespace("placeholder").getKey(), 0)
+                .put(S3Repository.RETRY_THROTTLED_DELETE_DELAY_INCREMENT.getKey(), TimeValue.timeValueMillis(10))
+                .put(S3Repository.RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES.getKey(), maxRetries)
+                .build(),
+            false
+        );
+        final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData);
+        final BlobContainer blobContainer = getBlobContainer(dataNodeName, repositoryName);
+        final TestTelemetryPlugin plugin = getPlugin(dataNodeName);
+        final int numberOfDeletes = randomIntBetween(1, 3);
+        final List<Long> numberOfRetriesPerAttempt = new ArrayList<>();
+        for (int i = 0; i < numberOfDeletes; i++) {
+            int numFailures = randomIntBetween(1, maxRetries);
+            numberOfRetriesPerAttempt.add((long) numFailures);
+            IntStream.range(0, numFailures).forEach(ignored -> addErrorStatus(S3_SLOW_DOWN_RESPONSE));
+            blobContainer.deleteBlobsIgnoringIfNotExists(
+                randomFrom(OperationPurpose.SNAPSHOT_DATA, OperationPurpose.SNAPSHOT_METADATA),
+                List.of(randomIdentifier()).iterator()
+            );
+        }
+        List<Measurement> longHistogramMeasurement = plugin.getLongHistogramMeasurement(METRIC_DELETE_RETRIES_HISTOGRAM);
+        assertThat(longHistogramMeasurement.stream().map(Measurement::getLong).toList(), equalTo(numberOfRetriesPerAttempt));
+    }
+
+    public void testRetrySnapshotDeleteMetricsWhenRetriesExhausted() {
+        final String repositoryName = randomRepositoryName();
+        // Disable retries in the client for this repo
+        int maxRetries = 3;
+        createRepository(
+            repositoryName,
+            Settings.builder()
+                .put(repositorySettings(repositoryName))
+                .put(S3ClientSettings.MAX_RETRIES_SETTING.getConcreteSettingForNamespace("placeholder").getKey(), 0)
+                .put(S3Repository.RETRY_THROTTLED_DELETE_DELAY_INCREMENT.getKey(), TimeValue.timeValueMillis(10))
+                .put(S3Repository.RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES.getKey(), maxRetries)
+                .build(),
+            false
+        );
+        final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData);
+        final BlobContainer blobContainer = getBlobContainer(dataNodeName, repositoryName);
+        final TestTelemetryPlugin plugin = getPlugin(dataNodeName);
+        // Keep throttling past the max number of retries
+        IntStream.range(0, maxRetries + 1).forEach(ignored -> addErrorStatus(S3_SLOW_DOWN_RESPONSE));
+        assertThrows(
+            IOException.class,
+            () -> blobContainer.deleteBlobsIgnoringIfNotExists(
+                randomFrom(OperationPurpose.SNAPSHOT_DATA, OperationPurpose.SNAPSHOT_METADATA),
+                List.of(randomIdentifier()).iterator()
+            )
+        );
+        List<Measurement> longHistogramMeasurement = plugin.getLongHistogramMeasurement(METRIC_DELETE_RETRIES_HISTOGRAM);
+        assertThat(longHistogramMeasurement.get(0).getLong(), equalTo(3L));
+    }
+
     private void addErrorStatus(RestStatus... statuses) {
-        errorStatusQueue.addAll(Arrays.asList(statuses));
+        errorResponseQueue.addAll(Arrays.stream(statuses).map(S3ErrorResponse::new).toList());
+    }
+
+    private void addErrorStatus(S3ErrorResponse... responses) {
+        errorResponseQueue.addAll(Arrays.asList(responses));
     }
 
     private long getLongCounterValue(TestTelemetryPlugin plugin, String instrumentName, Operation operation) {
@@ -275,25 +355,25 @@ private long getLongHistogramValue(TestTelemetryPlugin plugin, String instrument
     private static class S3MetricErroneousHttpHandler implements DelegatingHttpHandler {
 
         private final HttpHandler delegate;
-        private final Queue<RestStatus> errorStatusQueue;
+        private final Queue<S3ErrorResponse> errorResponseQueue;
 
-        S3MetricErroneousHttpHandler(HttpHandler delegate, Queue<RestStatus> errorStatusQueue) {
+        S3MetricErroneousHttpHandler(HttpHandler delegate, Queue<S3ErrorResponse> errorResponseQueue) {
             this.delegate = delegate;
-            this.errorStatusQueue = errorStatusQueue;
+            this.errorResponseQueue = errorResponseQueue;
         }
 
         @Override
         public void handle(HttpExchange exchange) throws IOException {
-            final RestStatus status = errorStatusQueue.poll();
-            if (status == null) {
+            final S3ErrorResponse errorResponse = errorResponseQueue.poll();
+            if (errorResponse == null) {
                 delegate.handle(exchange);
-            } else if (status == INTERNAL_SERVER_ERROR) {
+            } else if (errorResponse.status == INTERNAL_SERVER_ERROR) {
                 // Simulate an retryable exception
                 throw new IOException("ouch");
             } else {
                 try (exchange) {
                     drainInputStream(exchange.getRequestBody());
-                    exchange.sendResponseHeaders(status.getStatus(), -1);
+                    errorResponse.writeResponse(exchange);
                 }
             }
         }
@@ -302,4 +382,22 @@ public HttpHandler getDelegate() {
             return delegate;
         }
     }
+
+    record S3ErrorResponse(RestStatus status, String responseBody) {
+
+        S3ErrorResponse(RestStatus status) {
+            this(status, null);
+        }
+
+        @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint")
+        public void writeResponse(HttpExchange exchange) throws IOException {
+            if (responseBody != null) {
+                byte[] responseBytes = responseBody.getBytes(StandardCharsets.UTF_8);
+                exchange.sendResponseHeaders(status.getStatus(), responseBytes.length);
+                exchange.getResponseBody().write(responseBytes);
+            } else {
+                exchange.sendResponseHeaders(status.getStatus(), -1);
+            }
+        }
+    }
 }
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
index 49df078453327..9757d3af861a9 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.cluster.service.MasterService;
+import org.elasticsearch.common.BackoffPolicy;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -159,23 +160,22 @@ public void writeMetadataBlob(
     ) throws IOException {
         assert purpose != OperationPurpose.SNAPSHOT_DATA && BlobContainer.assertPurposeConsistency(purpose, blobName) : purpose;
         final String absoluteBlobKey = buildKey(blobName);
-        try (
-            AmazonS3Reference clientReference = blobStore.clientReference();
-            ChunkedBlobOutputStream<PartETag> out = new ChunkedBlobOutputStream<>(blobStore.bigArrays(), blobStore.bufferSizeInBytes()) {
+        try (ChunkedBlobOutputStream<PartETag> out = new ChunkedBlobOutputStream<>(blobStore.bigArrays(), blobStore.bufferSizeInBytes()) {
 
-                private final SetOnce<String> uploadId = new SetOnce<>();
+            private final SetOnce<String> uploadId = new SetOnce<>();
 
-                @Override
-                protected void flushBuffer() throws IOException {
-                    flushBuffer(false);
-                }
+            @Override
+            protected void flushBuffer() throws IOException {
+                flushBuffer(false);
+            }
 
-                private void flushBuffer(boolean lastPart) throws IOException {
-                    if (buffer.size() == 0) {
-                        return;
-                    }
-                    if (flushedBytes == 0L) {
-                        assert lastPart == false : "use single part upload if there's only a single part";
+            private void flushBuffer(boolean lastPart) throws IOException {
+                if (buffer.size() == 0) {
+                    return;
+                }
+                if (flushedBytes == 0L) {
+                    assert lastPart == false : "use single part upload if there's only a single part";
+                    try (AmazonS3Reference clientReference = blobStore.clientReference()) {
                         uploadId.set(
                             SocketAccess.doPrivileged(
                                 () -> clientReference.client()
@@ -183,51 +183,54 @@ private void flushBuffer(boolean lastPart) throws IOException {
                                     .getUploadId()
                             )
                         );
-                        if (Strings.isEmpty(uploadId.get())) {
-                            throw new IOException("Failed to initialize multipart upload " + absoluteBlobKey);
-                        }
                     }
-                    assert lastPart == false || successful : "must only write last part if successful";
-                    final UploadPartRequest uploadRequest = createPartUploadRequest(
-                        purpose,
-                        buffer.bytes().streamInput(),
-                        uploadId.get(),
-                        parts.size() + 1,
-                        absoluteBlobKey,
-                        buffer.size(),
-                        lastPart
-                    );
-                    final UploadPartResult uploadResponse = SocketAccess.doPrivileged(
-                        () -> clientReference.client().uploadPart(uploadRequest)
-                    );
-                    finishPart(uploadResponse.getPartETag());
+                    if (Strings.isEmpty(uploadId.get())) {
+                        throw new IOException("Failed to initialize multipart upload " + absoluteBlobKey);
+                    }
+                }
+                assert lastPart == false || successful : "must only write last part if successful";
+                final UploadPartRequest uploadRequest = createPartUploadRequest(
+                    purpose,
+                    buffer.bytes().streamInput(),
+                    uploadId.get(),
+                    parts.size() + 1,
+                    absoluteBlobKey,
+                    buffer.size(),
+                    lastPart
+                );
+                final UploadPartResult uploadResponse;
+                try (AmazonS3Reference clientReference = blobStore.clientReference()) {
+                    uploadResponse = SocketAccess.doPrivileged(() -> clientReference.client().uploadPart(uploadRequest));
                 }
+                finishPart(uploadResponse.getPartETag());
+            }
 
-                @Override
-                protected void onCompletion() throws IOException {
-                    if (flushedBytes == 0L) {
-                        writeBlob(purpose, blobName, buffer.bytes(), failIfAlreadyExists);
-                    } else {
-                        flushBuffer(true);
-                        final CompleteMultipartUploadRequest complRequest = new CompleteMultipartUploadRequest(
-                            blobStore.bucket(),
-                            absoluteBlobKey,
-                            uploadId.get(),
-                            parts
-                        );
-                        S3BlobStore.configureRequestForMetrics(complRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose);
+            @Override
+            protected void onCompletion() throws IOException {
+                if (flushedBytes == 0L) {
+                    writeBlob(purpose, blobName, buffer.bytes(), failIfAlreadyExists);
+                } else {
+                    flushBuffer(true);
+                    final CompleteMultipartUploadRequest complRequest = new CompleteMultipartUploadRequest(
+                        blobStore.bucket(),
+                        absoluteBlobKey,
+                        uploadId.get(),
+                        parts
+                    );
+                    S3BlobStore.configureRequestForMetrics(complRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose);
+                    try (AmazonS3Reference clientReference = blobStore.clientReference()) {
                         SocketAccess.doPrivilegedVoid(() -> clientReference.client().completeMultipartUpload(complRequest));
                     }
                 }
+            }
 
-                @Override
-                protected void onFailure() {
-                    if (Strings.hasText(uploadId.get())) {
-                        abortMultiPartUpload(purpose, uploadId.get(), absoluteBlobKey);
-                    }
+            @Override
+            protected void onFailure() {
+                if (Strings.hasText(uploadId.get())) {
+                    abortMultiPartUpload(purpose, uploadId.get(), absoluteBlobKey);
                 }
             }
-        ) {
+        }) {
             writer.accept(out);
             out.markSuccess();
         }
@@ -359,12 +362,9 @@ public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<St
 
     @Override
     public Map<String, BlobMetadata> listBlobsByPrefix(OperationPurpose purpose, @Nullable String blobNamePrefix) throws IOException {
-        try (AmazonS3Reference clientReference = blobStore.clientReference()) {
-            return executeListing(
-                purpose,
-                clientReference,
-                listObjectsRequest(purpose, blobNamePrefix == null ? keyPath : buildKey(blobNamePrefix))
-            ).stream()
+        try {
+            return executeListing(purpose, listObjectsRequest(purpose, blobNamePrefix == null ? keyPath : buildKey(blobNamePrefix)))
+                .stream()
                 .flatMap(listing -> listing.getObjectSummaries().stream())
                 .map(summary -> new BlobMetadata(summary.getKey().substring(keyPath.length()), summary.getSize()))
                 .collect(Collectors.toMap(BlobMetadata::name, Function.identity()));
@@ -380,8 +380,8 @@ public Map<String, BlobMetadata> listBlobs(OperationPurpose purpose) throws IOEx
 
     @Override
     public Map<String, BlobContainer> children(OperationPurpose purpose) throws IOException {
-        try (AmazonS3Reference clientReference = blobStore.clientReference()) {
-            return executeListing(purpose, clientReference, listObjectsRequest(purpose, keyPath)).stream().flatMap(listing -> {
+        try {
+            return executeListing(purpose, listObjectsRequest(purpose, keyPath)).stream().flatMap(listing -> {
                 assert listing.getObjectSummaries().stream().noneMatch(s -> {
                     for (String commonPrefix : listing.getCommonPrefixes()) {
                         if (s.getKey().substring(keyPath.length()).startsWith(commonPrefix)) {
@@ -402,21 +402,19 @@ public Map<String, BlobContainer> children(OperationPurpose purpose) throws IOEx
         }
     }
 
-    private List<ObjectListing> executeListing(
-        OperationPurpose purpose,
-        AmazonS3Reference clientReference,
-        ListObjectsRequest listObjectsRequest
-    ) {
+    private List<ObjectListing> executeListing(OperationPurpose purpose, ListObjectsRequest listObjectsRequest) {
         final List<ObjectListing> results = new ArrayList<>();
         ObjectListing prevListing = null;
         while (true) {
             ObjectListing list;
-            if (prevListing != null) {
-                final var listNextBatchOfObjectsRequest = new ListNextBatchOfObjectsRequest(prevListing);
-                S3BlobStore.configureRequestForMetrics(listNextBatchOfObjectsRequest, blobStore, Operation.LIST_OBJECTS, purpose);
-                list = SocketAccess.doPrivileged(() -> clientReference.client().listNextBatchOfObjects(listNextBatchOfObjectsRequest));
-            } else {
-                list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest));
+            try (AmazonS3Reference clientReference = blobStore.clientReference()) {
+                if (prevListing != null) {
+                    final var listNextBatchOfObjectsRequest = new ListNextBatchOfObjectsRequest(prevListing);
+                    S3BlobStore.configureRequestForMetrics(listNextBatchOfObjectsRequest, blobStore, Operation.LIST_OBJECTS, purpose);
+                    list = SocketAccess.doPrivileged(() -> clientReference.client().listNextBatchOfObjects(listNextBatchOfObjectsRequest));
+                } else {
+                    list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest));
+                }
             }
             results.add(list);
             if (list.isTruncated()) {
@@ -503,13 +501,14 @@ void executeMultipartUpload(
         final SetOnce<String> uploadId = new SetOnce<>();
         final String bucketName = s3BlobStore.bucket();
         boolean success = false;
-        try (AmazonS3Reference clientReference = s3BlobStore.clientReference()) {
-
-            uploadId.set(
-                SocketAccess.doPrivileged(
-                    () -> clientReference.client().initiateMultipartUpload(initiateMultiPartUpload(purpose, blobName)).getUploadId()
-                )
-            );
+        try {
+            try (AmazonS3Reference clientReference = s3BlobStore.clientReference()) {
+                uploadId.set(
+                    SocketAccess.doPrivileged(
+                        () -> clientReference.client().initiateMultipartUpload(initiateMultiPartUpload(purpose, blobName)).getUploadId()
+                    )
+                );
+            }
             if (Strings.isEmpty(uploadId.get())) {
                 throw new IOException("Failed to initialize multipart upload " + blobName);
             }
@@ -530,8 +529,12 @@ void executeMultipartUpload(
                 );
                 bytesCount += uploadRequest.getPartSize();
 
-                final UploadPartResult uploadResponse = SocketAccess.doPrivileged(() -> clientReference.client().uploadPart(uploadRequest));
-                parts.add(uploadResponse.getPartETag());
+                try (AmazonS3Reference clientReference = s3BlobStore.clientReference()) {
+                    final UploadPartResult uploadResponse = SocketAccess.doPrivileged(
+                        () -> clientReference.client().uploadPart(uploadRequest)
+                    );
+                    parts.add(uploadResponse.getPartETag());
+                }
             }
 
             if (bytesCount != blobSize) {
@@ -547,7 +550,9 @@ void executeMultipartUpload(
                 parts
             );
             S3BlobStore.configureRequestForMetrics(complRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose);
-            SocketAccess.doPrivilegedVoid(() -> clientReference.client().completeMultipartUpload(complRequest));
+            try (AmazonS3Reference clientReference = s3BlobStore.clientReference()) {
+                SocketAccess.doPrivilegedVoid(() -> clientReference.client().completeMultipartUpload(complRequest));
+            }
             success = true;
 
         } catch (final AmazonClientException e) {
@@ -910,21 +915,44 @@ public void compareAndExchangeRegister(
     @Override
     public void getRegister(OperationPurpose purpose, String key, ActionListener<OptionalBytesReference> listener) {
         ActionListener.completeWith(listener, () -> {
-            final var getObjectRequest = new GetObjectRequest(blobStore.bucket(), buildKey(key));
-            S3BlobStore.configureRequestForMetrics(getObjectRequest, blobStore, Operation.GET_OBJECT, purpose);
-            try (
-                var clientReference = blobStore.clientReference();
-                var s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest));
-                var stream = s3Object.getObjectContent()
-            ) {
-                return OptionalBytesReference.of(getRegisterUsingConsistentRead(stream, keyPath, key));
-            } catch (AmazonS3Exception e) {
-                logger.trace(() -> Strings.format("[%s]: getRegister failed", key), e);
-                if (e.getStatusCode() == 404) {
-                    return OptionalBytesReference.EMPTY;
-                } else {
-                    throw e;
+            final var backoffPolicy = purpose == OperationPurpose.REPOSITORY_ANALYSIS
+                ? BackoffPolicy.noBackoff()
+                : BackoffPolicy.constantBackoff(blobStore.getGetRegisterRetryDelay(), blobStore.getMaxRetries());
+            final var retryDelayIterator = backoffPolicy.iterator();
+
+            Exception finalException = null;
+            while (true) {
+                final var getObjectRequest = new GetObjectRequest(blobStore.bucket(), buildKey(key));
+                S3BlobStore.configureRequestForMetrics(getObjectRequest, blobStore, Operation.GET_OBJECT, purpose);
+                try (
+                    var clientReference = blobStore.clientReference();
+                    var s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest));
+                    var stream = s3Object.getObjectContent()
+                ) {
+                    return OptionalBytesReference.of(getRegisterUsingConsistentRead(stream, keyPath, key));
+                } catch (Exception attemptException) {
+                    logger.trace(() -> Strings.format("[%s]: getRegister failed", key), attemptException);
+                    if (attemptException instanceof AmazonS3Exception amazonS3Exception && amazonS3Exception.getStatusCode() == 404) {
+                        return OptionalBytesReference.EMPTY;
+                    } else if (finalException == null) {
+                        finalException = attemptException;
+                    } else if (finalException != attemptException) {
+                        finalException.addSuppressed(attemptException);
+                    }
+                }
+                if (retryDelayIterator.hasNext()) {
+                    try {
+                        // noinspection BusyWait
+                        Thread.sleep(retryDelayIterator.next().millis());
+                        continue;
+                    } catch (InterruptedException interruptedException) {
+                        Thread.currentThread().interrupt();
+                        finalException.addSuppressed(interruptedException);
+                        // fall through and throw the exception
+                    }
                 }
+
+                throw finalException;
             }
         });
     }
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
index 3e6b7c356cb11..5fb3254df819b 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
@@ -14,6 +14,7 @@
 import com.amazonaws.Request;
 import com.amazonaws.Response;
 import com.amazonaws.metrics.RequestMetricCollector;
+import com.amazonaws.retry.RetryUtils;
 import com.amazonaws.services.s3.model.CannedAccessControlList;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
@@ -25,6 +26,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
+import org.elasticsearch.common.BackoffPolicy;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -91,6 +93,9 @@ class S3BlobStore implements BlobStore {
     private final StatsCollectors statsCollectors = new StatsCollectors();
 
     private final int bulkDeletionBatchSize;
+    private final BackoffPolicy retryThrottledDeleteBackoffPolicy;
+
+    private final TimeValue getRegisterRetryDelay;
 
     S3BlobStore(
         S3Service service,
@@ -102,7 +107,8 @@ class S3BlobStore implements BlobStore {
         RepositoryMetadata repositoryMetadata,
         BigArrays bigArrays,
         ThreadPool threadPool,
-        S3RepositoriesMetrics s3RepositoriesMetrics
+        S3RepositoriesMetrics s3RepositoriesMetrics,
+        BackoffPolicy retryThrottledDeleteBackoffPolicy
     ) {
         this.service = service;
         this.bigArrays = bigArrays;
@@ -116,7 +122,8 @@ class S3BlobStore implements BlobStore {
         this.snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
         this.s3RepositoriesMetrics = s3RepositoriesMetrics;
         this.bulkDeletionBatchSize = S3Repository.DELETION_BATCH_SIZE_SETTING.get(repositoryMetadata.settings());
-
+        this.retryThrottledDeleteBackoffPolicy = retryThrottledDeleteBackoffPolicy;
+        this.getRegisterRetryDelay = S3Repository.GET_REGISTER_RETRY_DELAY.get(repositoryMetadata.settings());
     }
 
     RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) {
@@ -255,7 +262,8 @@ private boolean assertConsistencyBetweenHttpRequestAndOperation(Request<?> reque
     private static long getCountForMetric(TimingInfo info, AWSRequestMetrics.Field field) {
         var count = info.getCounter(field.name());
         if (count == null) {
-            if (field == AWSRequestMetrics.Field.RequestCount) {
+            // This can be null if the thread was interrupted
+            if (field == AWSRequestMetrics.Field.RequestCount && Thread.currentThread().isInterrupted() == false) {
                 final String message = "Expected request count to be tracked but found not count.";
                 assert false : message;
                 logger.warn(message);
@@ -331,18 +339,18 @@ public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<St
         }
 
         final List<String> partition = new ArrayList<>();
-        try (AmazonS3Reference clientReference = clientReference()) {
+        try {
             // S3 API only allows 1k blobs per delete so we split up the given blobs into requests of max. 1k deletes
             final AtomicReference<Exception> aex = new AtomicReference<>();
             blobNames.forEachRemaining(key -> {
                 partition.add(key);
                 if (partition.size() == bulkDeletionBatchSize) {
-                    deletePartition(purpose, clientReference, partition, aex);
+                    deletePartition(purpose, partition, aex);
                     partition.clear();
                 }
             });
             if (partition.isEmpty() == false) {
-                deletePartition(purpose, clientReference, partition, aex);
+                deletePartition(purpose, partition, aex);
             }
             if (aex.get() != null) {
                 throw aex.get();
@@ -352,30 +360,84 @@ public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<St
         }
     }
 
-    private void deletePartition(
-        OperationPurpose purpose,
-        AmazonS3Reference clientReference,
-        List<String> partition,
-        AtomicReference<Exception> aex
-    ) {
-        try {
-            SocketAccess.doPrivilegedVoid(() -> clientReference.client().deleteObjects(bulkDelete(purpose, this, partition)));
-        } catch (MultiObjectDeleteException e) {
-            // We are sending quiet mode requests so we can't use the deleted keys entry on the exception and instead
-            // first remove all keys that were sent in the request and then add back those that ran into an exception.
+    /**
+     * Delete one partition of a batch of blobs
+     *
+     * @param purpose The {@link OperationPurpose} of the deletion
+     * @param partition The list of blobs to delete
+     * @param aex A holder for any exception(s) thrown during the deletion
+     */
+    private void deletePartition(OperationPurpose purpose, List<String> partition, AtomicReference<Exception> aex) {
+        final Iterator<TimeValue> retries = retryThrottledDeleteBackoffPolicy.iterator();
+        int retryCounter = 0;
+        while (true) {
+            try (AmazonS3Reference clientReference = clientReference()) {
+                SocketAccess.doPrivilegedVoid(() -> clientReference.client().deleteObjects(bulkDelete(purpose, this, partition)));
+                s3RepositoriesMetrics.retryDeletesHistogram().record(retryCounter);
+                return;
+            } catch (MultiObjectDeleteException e) {
+                // We are sending quiet mode requests so we can't use the deleted keys entry on the exception and instead
+                // first remove all keys that were sent in the request and then add back those that ran into an exception.
+                logger.warn(
+                    () -> format(
+                        "Failed to delete some blobs %s",
+                        e.getErrors()
+                            .stream()
+                            .map(err -> "[" + err.getKey() + "][" + err.getCode() + "][" + err.getMessage() + "]")
+                            .toList()
+                    ),
+                    e
+                );
+                aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e));
+                return;
+            } catch (AmazonClientException e) {
+                if (shouldRetryDelete(purpose) && RetryUtils.isThrottlingException(e)) {
+                    // S3 is asking us to slow down. Pause for a bit and retry
+                    if (maybeDelayAndRetryDelete(retries)) {
+                        retryCounter++;
+                    } else {
+                        s3RepositoriesMetrics.retryDeletesHistogram().record(retryCounter);
+                        aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e));
+                        return;
+                    }
+                } else {
+                    // The AWS client threw any unexpected exception and did not execute the request at all so we do not
+                    // remove any keys from the outstanding deletes set.
+                    aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e));
+                    return;
+                }
+            }
+        }
+    }
+
+    /**
+     * If there are remaining retries, pause for the configured interval then return true
+     *
+     * @param retries The retries iterator
+     * @return true to try the deletion again, false otherwise
+     */
+    private boolean maybeDelayAndRetryDelete(Iterator<TimeValue> retries) {
+        if (retries.hasNext()) {
+            try {
+                Thread.sleep(retries.next().millis());
+                return true;
+            } catch (InterruptedException iex) {
+                Thread.currentThread().interrupt();
+                // If we're interrupted, record the exception and abort retries
+                logger.warn("Aborting tenacious snapshot delete retries due to interrupt");
+            }
+        } else {
             logger.warn(
-                () -> format(
-                    "Failed to delete some blobs %s",
-                    e.getErrors().stream().map(err -> "[" + err.getKey() + "][" + err.getCode() + "][" + err.getMessage() + "]").toList()
-                ),
-                e
+                "Exceeded maximum tenacious snapshot delete retries, aborting. Using back-off policy "
+                    + retryThrottledDeleteBackoffPolicy
+                    + ", see the throttled_delete_retry.* S3 repository properties to configure the back-off parameters"
             );
-            aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e));
-        } catch (AmazonClientException e) {
-            // The AWS client threw any unexpected exception and did not execute the request at all so we do not
-            // remove any keys from the outstanding deletes set.
-            aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e));
         }
+        return false;
+    }
+
+    private boolean shouldRetryDelete(OperationPurpose operationPurpose) {
+        return operationPurpose == OperationPurpose.SNAPSHOT_DATA || operationPurpose == OperationPurpose.SNAPSHOT_METADATA;
     }
 
     private static DeleteObjectsRequest bulkDelete(OperationPurpose purpose, S3BlobStore blobStore, List<String> blobs) {
@@ -409,6 +471,10 @@ public StorageClass getStorageClass() {
         return storageClass;
     }
 
+    public TimeValue getGetRegisterRetryDelay() {
+        return getRegisterRetryDelay;
+    }
+
     public static StorageClass initStorageClass(String storageClass) {
         if ((storageClass == null) || storageClass.equals("")) {
             return StorageClass.Standard;
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java
index 74682ca190a0c..03106c26c9a29 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java
@@ -17,7 +17,8 @@ public record S3RepositoriesMetrics(
     RepositoriesMetrics common,
     LongCounter retryStartedCounter,
     LongCounter retryCompletedCounter,
-    LongHistogram retryHistogram
+    LongHistogram retryHistogram,
+    LongHistogram retryDeletesHistogram
 ) {
 
     public static S3RepositoriesMetrics NOOP = new S3RepositoriesMetrics(RepositoriesMetrics.NOOP);
@@ -25,6 +26,7 @@ public record S3RepositoriesMetrics(
     public static final String METRIC_RETRY_EVENT_TOTAL = "es.repositories.s3.input_stream.retry.event.total";
     public static final String METRIC_RETRY_SUCCESS_TOTAL = "es.repositories.s3.input_stream.retry.success.total";
     public static final String METRIC_RETRY_ATTEMPTS_HISTOGRAM = "es.repositories.s3.input_stream.retry.attempts.histogram";
+    public static final String METRIC_DELETE_RETRIES_HISTOGRAM = "es.repositories.s3.delete.retry.attempts.histogram";
 
     public S3RepositoriesMetrics(RepositoriesMetrics common) {
         this(
@@ -32,7 +34,8 @@ public S3RepositoriesMetrics(RepositoriesMetrics common) {
             common.meterRegistry().registerLongCounter(METRIC_RETRY_EVENT_TOTAL, "s3 input stream retry event count", "unit"),
             common.meterRegistry().registerLongCounter(METRIC_RETRY_SUCCESS_TOTAL, "s3 input stream retry success count", "unit"),
             common.meterRegistry()
-                .registerLongHistogram(METRIC_RETRY_ATTEMPTS_HISTOGRAM, "s3 input stream retry attempts histogram", "unit")
+                .registerLongHistogram(METRIC_RETRY_ATTEMPTS_HISTOGRAM, "s3 input stream retry attempts histogram", "unit"),
+            common.meterRegistry().registerLongHistogram(METRIC_DELETE_RETRIES_HISTOGRAM, "s3 delete retry attempts histogram", "unit")
         );
     }
 }
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index af385eeac6a5b..fde15d5d6e6bc 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.BackoffPolicy;
 import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -202,6 +203,36 @@ class S3Repository extends MeteredBlobStoreRepository {
         Setting.Property.Dynamic
     );
 
+    /**
+     * We will retry deletes that fail due to throttling. We use an {@link BackoffPolicy#linearBackoff(TimeValue, int, TimeValue)}
+     * with the following parameters
+     */
+    static final Setting<TimeValue> RETRY_THROTTLED_DELETE_DELAY_INCREMENT = Setting.timeSetting(
+        "throttled_delete_retry.delay_increment",
+        TimeValue.timeValueMillis(50),
+        TimeValue.ZERO
+    );
+    static final Setting<TimeValue> RETRY_THROTTLED_DELETE_MAXIMUM_DELAY = Setting.timeSetting(
+        "throttled_delete_retry.maximum_delay",
+        TimeValue.timeValueSeconds(5),
+        TimeValue.ZERO
+    );
+    static final Setting<Integer> RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES = Setting.intSetting(
+        "throttled_delete_retry.maximum_number_of_retries",
+        10,
+        0
+    );
+
+    /**
+     * Time to wait before trying again if getRegister fails.
+     */
+    static final Setting<TimeValue> GET_REGISTER_RETRY_DELAY = Setting.timeSetting(
+        "get_register_retry_delay",
+        new TimeValue(5, TimeUnit.SECONDS),
+        new TimeValue(0, TimeUnit.MILLISECONDS),
+        Setting.Property.Dynamic
+    );
+
     private final S3Service service;
 
     private final String bucket;
@@ -424,7 +455,12 @@ protected S3BlobStore createBlobStore() {
             metadata,
             bigArrays,
             threadPool,
-            s3RepositoriesMetrics
+            s3RepositoriesMetrics,
+            BackoffPolicy.linearBackoff(
+                RETRY_THROTTLED_DELETE_DELAY_INCREMENT.get(metadata.settings()),
+                RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES.get(metadata.settings()),
+                RETRY_THROTTLED_DELETE_MAXIMUM_DELAY.get(metadata.settings())
+            )
         );
     }
 
diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
index 1443ff704efd1..b292dc5872994 100644
--- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -10,10 +10,12 @@
 
 import fixture.s3.S3HttpHandler;
 
+import com.amazonaws.AbortedException;
 import com.amazonaws.DnsResolver;
 import com.amazonaws.SdkClientException;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.internal.MD5DigestCalculatingInputStream;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.util.Base16;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
@@ -21,9 +23,12 @@
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
+import org.elasticsearch.common.BackoffPolicy;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.OperationPurpose;
+import org.elasticsearch.common.blobstore.OptionalBytesReference;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
@@ -62,15 +67,18 @@
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.OptionalInt;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.IntConsumer;
 
 import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose;
 import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose;
@@ -98,6 +106,7 @@
 @SuppressForbidden(reason = "use a http server")
 public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTestCase {
 
+    private static final int MAX_NUMBER_SNAPSHOT_DELETE_RETRIES = 10;
     private S3Service service;
     private AtomicBoolean shouldErrorOnDns;
     private RecordingMeterRegistry recordingMeterRegistry;
@@ -183,7 +192,10 @@ protected BlobContainer createBlobContainer(
         final RepositoryMetadata repositoryMetadata = new RepositoryMetadata(
             "repository",
             S3Repository.TYPE,
-            Settings.builder().put(S3Repository.CLIENT_NAME.getKey(), clientName).build()
+            Settings.builder()
+                .put(S3Repository.CLIENT_NAME.getKey(), clientName)
+                .put(S3Repository.GET_REGISTER_RETRY_DELAY.getKey(), TimeValue.ZERO)
+                .build()
         );
 
         final S3BlobStore s3BlobStore = new S3BlobStore(
@@ -196,7 +208,8 @@ protected BlobContainer createBlobContainer(
             repositoryMetadata,
             BigArrays.NON_RECYCLING_INSTANCE,
             new DeterministicTaskQueue().getThreadPool(),
-            new S3RepositoriesMetrics(new RepositoriesMetrics(recordingMeterRegistry))
+            new S3RepositoriesMetrics(new RepositoriesMetrics(recordingMeterRegistry)),
+            BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(1), MAX_NUMBER_SNAPSHOT_DELETE_RETRIES)
         );
         return new S3BlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), s3BlobStore) {
             @Override
@@ -573,16 +586,16 @@ public void handle(HttpExchange exchange) throws IOException {
                         ),
                         -1
                     );
+                    exchange.getResponseBody().flush();
                 } else if (randomBoolean()) {
                     final var bytesSent = sendIncompleteContent(exchange, bytes);
                     if (bytesSent < meaningfulProgressBytes) {
                         failuresWithoutProgress += 1;
-                    } else {
-                        exchange.getResponseBody().flush();
                     }
                 } else {
                     failuresWithoutProgress += 1;
                 }
+                exchange.getResponseBody().flush();
                 exchange.close();
             }
         }
@@ -627,6 +640,7 @@ public void handle(HttpExchange exchange) throws IOException {
                 failureCount += 1;
                 Streams.readFully(exchange.getRequestBody());
                 sendIncompleteContent(exchange, bytes);
+                exchange.getResponseBody().flush();
                 exchange.close();
             }
         }
@@ -771,6 +785,240 @@ public void handle(HttpExchange exchange) throws IOException {
         assertThat(getRetryHistogramMeasurements(), empty());
     }
 
+    public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
+        // disable AWS-client retries
+        final BlobContainer blobContainer = createBlobContainer(0, null, true, null);
+
+        int numBlobsToDelete = randomIntBetween(500, 3000);
+        List<String> blobsToDelete = new ArrayList<>();
+        for (int i = 0; i < numBlobsToDelete; i++) {
+            blobsToDelete.add(randomIdentifier());
+        }
+        int throttleTimesBeforeSuccess = randomIntBetween(1, MAX_NUMBER_SNAPSHOT_DELETE_RETRIES);
+        logger.info("--> Throttling {} times before success", throttleTimesBeforeSuccess);
+        ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(throttleTimesBeforeSuccess, attempt -> {});
+        httpServer.createContext("/", handler);
+        blobContainer.deleteBlobsIgnoringIfNotExists(randomFrom(operationPurposesThatRetryOnDelete()), blobsToDelete.iterator());
+
+        int expectedNumberOfBatches = expectedNumberOfBatches(numBlobsToDelete);
+        assertThat(handler.numberOfDeleteAttempts.get(), equalTo(throttleTimesBeforeSuccess + expectedNumberOfBatches));
+        assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(expectedNumberOfBatches));
+    }
+
+    public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
+        // disable AWS-client retries
+        final BlobContainer blobContainer = createBlobContainer(0, null, true, null);
+
+        int numBlobsToDelete = randomIntBetween(500, 3000);
+        List<String> blobsToDelete = new ArrayList<>();
+        for (int i = 0; i < numBlobsToDelete; i++) {
+            blobsToDelete.add(randomIdentifier());
+        }
+
+        final Thread clientThread = Thread.currentThread();
+        int interruptBeforeAttempt = randomIntBetween(0, randomIntBetween(1, 10));
+        logger.info("--> Deleting {} blobs, interrupting before attempt {}", numBlobsToDelete, interruptBeforeAttempt);
+        ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(Integer.MAX_VALUE, attempt -> {
+            if (attempt == interruptBeforeAttempt) {
+                clientThread.interrupt();
+            }
+        });
+        httpServer.createContext("/", handler);
+
+        try {
+            IOException exception = assertThrows(
+                IOException.class,
+                () -> blobContainer.deleteBlobsIgnoringIfNotExists(
+                    randomFrom(operationPurposesThatRetryOnDelete()),
+                    blobsToDelete.iterator()
+                )
+            );
+            assertThat(exception.getCause(), instanceOf(AbortedException.class));
+            assertThat(handler.numberOfDeleteAttempts.get(), equalTo(interruptBeforeAttempt + 1));
+            assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(0));
+        } finally {
+            // interrupt should be preserved, clear it to prevent it leaking between tests
+            assertTrue(Thread.interrupted());
+        }
+    }
+
+    public void testNonSnapshotDeletesAreNotRetried() {
+        // disable AWS-client retries
+        final BlobContainer blobContainer = createBlobContainer(0, null, true, null);
+
+        int numBlobsToDelete = randomIntBetween(500, 3000);
+        List<String> blobsToDelete = new ArrayList<>();
+        for (int i = 0; i < numBlobsToDelete; i++) {
+            blobsToDelete.add(randomIdentifier());
+        }
+        ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(Integer.MAX_VALUE, attempt -> {});
+        httpServer.createContext("/", handler);
+        IOException exception = assertThrows(
+            IOException.class,
+            () -> blobContainer.deleteBlobsIgnoringIfNotExists(
+                randomValueOtherThanMany(
+                    op -> operationPurposesThatRetryOnDelete().contains(op),
+                    () -> randomFrom(OperationPurpose.values())
+                ),
+                blobsToDelete.iterator()
+            )
+        );
+        assertEquals(
+            ThrottlingDeleteHandler.THROTTLING_ERROR_CODE,
+            asInstanceOf(AmazonS3Exception.class, exception.getCause()).getErrorCode()
+        );
+        assertThat(handler.numberOfDeleteAttempts.get(), equalTo(expectedNumberOfBatches(numBlobsToDelete)));
+        assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(0));
+    }
+
+    public void testNonThrottlingErrorsAreNotRetried() {
+        // disable AWS-client retries
+        final BlobContainer blobContainer = createBlobContainer(0, null, true, null);
+
+        int numBlobsToDelete = randomIntBetween(500, 3000);
+        List<String> blobsToDelete = new ArrayList<>();
+        for (int i = 0; i < numBlobsToDelete; i++) {
+            blobsToDelete.add(randomIdentifier());
+        }
+        ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(Integer.MAX_VALUE, attempt -> {}, "NotThrottling");
+        httpServer.createContext("/", handler);
+        assertThrows(
+            IOException.class,
+            () -> blobContainer.deleteBlobsIgnoringIfNotExists(randomFrom(operationPurposesThatRetryOnDelete()), blobsToDelete.iterator())
+        );
+        assertThat(handler.numberOfDeleteAttempts.get(), equalTo(expectedNumberOfBatches(numBlobsToDelete)));
+        assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(0));
+    }
+
+    private int expectedNumberOfBatches(int blobsToDelete) {
+        return (blobsToDelete / 1_000) + (blobsToDelete % 1_000 == 0 ? 0 : 1);
+    }
+
+    @SuppressForbidden(reason = "use a http server")
+    private class ThrottlingDeleteHandler extends S3HttpHandler {
+
+        private static final String THROTTLING_ERROR_CODE = "SlowDown";
+
+        private final AtomicInteger throttleTimesBeforeSuccess;
+        private final AtomicInteger numberOfDeleteAttempts;
+        private final AtomicInteger numberOfSuccessfulDeletes;
+        private final IntConsumer onAttemptCallback;
+        private final String errorCode;
+
+        ThrottlingDeleteHandler(int throttleTimesBeforeSuccess, IntConsumer onAttemptCallback) {
+            this(throttleTimesBeforeSuccess, onAttemptCallback, THROTTLING_ERROR_CODE);
+        }
+
+        ThrottlingDeleteHandler(int throttleTimesBeforeSuccess, IntConsumer onAttemptCallback, String errorCode) {
+            super("bucket");
+            this.numberOfDeleteAttempts = new AtomicInteger();
+            this.numberOfSuccessfulDeletes = new AtomicInteger();
+            this.throttleTimesBeforeSuccess = new AtomicInteger(throttleTimesBeforeSuccess);
+            this.onAttemptCallback = onAttemptCallback;
+            this.errorCode = errorCode;
+        }
+
+        @Override
+        public void handle(HttpExchange exchange) throws IOException {
+            if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) {
+                onAttemptCallback.accept(numberOfDeleteAttempts.get());
+                numberOfDeleteAttempts.incrementAndGet();
+                if (throttleTimesBeforeSuccess.getAndDecrement() > 0) {
+                    final byte[] responseBytes = Strings.format("""
+                        <?xml version="1.0" encoding="UTF-8"?>
+                        <Error>
+                          <Code>%s</Code>
+                          <Message>This is a throttling message</Message>
+                          <Resource>/bucket/</Resource>
+                          <RequestId>4442587FB7D0A2F9</RequestId>
+                        </Error>""", errorCode).getBytes(StandardCharsets.UTF_8);
+
+                    exchange.sendResponseHeaders(HttpStatus.SC_SERVICE_UNAVAILABLE, responseBytes.length);
+                    exchange.getResponseBody().write(responseBytes);
+                    exchange.close();
+                } else {
+                    numberOfSuccessfulDeletes.incrementAndGet();
+                    super.handle(exchange);
+                }
+            } else {
+                super.handle(exchange);
+            }
+        }
+    }
+
+    private Set<OperationPurpose> operationPurposesThatRetryOnDelete() {
+        return Set.of(OperationPurpose.SNAPSHOT_DATA, OperationPurpose.SNAPSHOT_METADATA);
+    }
+
+    public void testGetRegisterRetries() {
+        final var maxRetries = between(0, 3);
+        final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null);
+
+        interface FailingHandlerFactory {
+            void addHandler(String blobName, Integer... responseCodes);
+        }
+
+        final var requestCounter = new AtomicInteger();
+        final FailingHandlerFactory countingFailingHandlerFactory = (blobName, responseCodes) -> httpServer.createContext(
+            downloadStorageEndpoint(blobContainer, blobName),
+            exchange -> {
+                requestCounter.incrementAndGet();
+                try (exchange) {
+                    exchange.sendResponseHeaders(randomFrom(responseCodes), -1);
+                }
+            }
+        );
+
+        countingFailingHandlerFactory.addHandler("test_register_no_internal_retries", HttpStatus.SC_UNPROCESSABLE_ENTITY);
+        countingFailingHandlerFactory.addHandler(
+            "test_register_internal_retries",
+            HttpStatus.SC_INTERNAL_SERVER_ERROR,
+            HttpStatus.SC_SERVICE_UNAVAILABLE
+        );
+        countingFailingHandlerFactory.addHandler("test_register_not_found", HttpStatus.SC_NOT_FOUND);
+
+        {
+            final var exceptionWithInternalRetries = safeAwaitFailure(
+                OptionalBytesReference.class,
+                l -> blobContainer.getRegister(randomRetryingPurpose(), "test_register_internal_retries", l)
+            );
+            assertThat(exceptionWithInternalRetries, instanceOf(AmazonS3Exception.class));
+            assertEquals((maxRetries + 1) * (maxRetries + 1), requestCounter.get());
+            assertEquals(maxRetries, exceptionWithInternalRetries.getSuppressed().length);
+        }
+
+        {
+            requestCounter.set(0);
+            final var exceptionWithoutInternalRetries = safeAwaitFailure(
+                OptionalBytesReference.class,
+                l -> blobContainer.getRegister(randomRetryingPurpose(), "test_register_no_internal_retries", l)
+            );
+            assertThat(exceptionWithoutInternalRetries, instanceOf(AmazonS3Exception.class));
+            assertEquals(maxRetries + 1, requestCounter.get());
+            assertEquals(maxRetries, exceptionWithoutInternalRetries.getSuppressed().length);
+        }
+
+        {
+            requestCounter.set(0);
+            final var repoAnalysisException = safeAwaitFailure(
+                OptionalBytesReference.class,
+                l -> blobContainer.getRegister(OperationPurpose.REPOSITORY_ANALYSIS, "test_register_no_internal_retries", l)
+            );
+            assertThat(repoAnalysisException, instanceOf(AmazonS3Exception.class));
+            assertEquals(1, requestCounter.get());
+            assertEquals(0, repoAnalysisException.getSuppressed().length);
+        }
+
+        {
+            requestCounter.set(0);
+            final OptionalBytesReference expectEmpty = safeAwait(
+                l -> blobContainer.getRegister(randomPurpose(), "test_register_not_found", l)
+            );
+            assertEquals(OptionalBytesReference.EMPTY, expectEmpty);
+            assertEquals(1, requestCounter.get());
+        }
+    }
+
     @Override
     protected Matcher<Integer> getMaxRetriesMatcher(int maxRetries) {
         // some attempts make meaningful progress and do not count towards the max retry limit
diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java
index f52b3f4b53a62..58bb11874fbe6 100644
--- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java
+++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java
@@ -114,9 +114,7 @@ public void testExecuteSingleUpload() throws IOException {
             when(blobStore.getCannedACL()).thenReturn(cannedAccessControlList);
         }
 
-        final AmazonS3 client = mock(AmazonS3.class);
-        final AmazonS3Reference clientReference = new AmazonS3Reference(client);
-        when(blobStore.clientReference()).thenReturn(clientReference);
+        final AmazonS3 client = configureMockClient(blobStore);
 
         final ArgumentCaptor<PutObjectRequest> argumentCaptor = ArgumentCaptor.forClass(PutObjectRequest.class);
         when(client.putObject(argumentCaptor.capture())).thenReturn(new PutObjectResult());
@@ -187,9 +185,7 @@ public void testExecuteMultipartUpload() throws IOException {
             when(blobStore.getCannedACL()).thenReturn(cannedAccessControlList);
         }
 
-        final AmazonS3 client = mock(AmazonS3.class);
-        final AmazonS3Reference clientReference = new AmazonS3Reference(client);
-        when(blobStore.clientReference()).thenReturn(clientReference);
+        final AmazonS3 client = configureMockClient(blobStore);
 
         final ArgumentCaptor<InitiateMultipartUploadRequest> initArgCaptor = ArgumentCaptor.forClass(InitiateMultipartUploadRequest.class);
         final InitiateMultipartUploadResult initResult = new InitiateMultipartUploadResult();
@@ -260,6 +256,8 @@ public void testExecuteMultipartUpload() throws IOException {
 
         final List<String> actualETags = compRequest.getPartETags().stream().map(PartETag::getETag).collect(Collectors.toList());
         assertEquals(expectedEtags, actualETags);
+
+        closeMockClient(blobStore);
     }
 
     public void testExecuteMultipartUploadAborted() {
@@ -356,6 +354,27 @@ public void testExecuteMultipartUploadAborted() {
             assertEquals(blobName, abortRequest.getKey());
             assertEquals(uploadId, abortRequest.getUploadId());
         }
+
+        closeMockClient(blobStore);
+    }
+
+    private static AmazonS3 configureMockClient(S3BlobStore blobStore) {
+        final AmazonS3 client = mock(AmazonS3.class);
+        try (AmazonS3Reference clientReference = new AmazonS3Reference(client)) {
+            clientReference.mustIncRef(); // held by the mock, ultimately released in closeMockClient
+            when(blobStore.clientReference()).then(invocation -> {
+                clientReference.mustIncRef();
+                return clientReference;
+            });
+        }
+        return client;
+    }
+
+    private static void closeMockClient(S3BlobStore blobStore) {
+        final var finalClientReference = blobStore.clientReference();
+        assertFalse(finalClientReference.decRef());
+        assertTrue(finalClientReference.decRef());
+        assertFalse(finalClientReference.hasReferences());
     }
 
     public void testNumberOfMultipartsWithZeroPartSize() {
diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java
index 26d31b941f356..b5c272f41a1d5 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java
@@ -175,12 +175,16 @@ public void testClientConnectionCloseMidStream() throws Exception {
             var handler = ctx.awaitRestChannelAccepted(opaqueId);
             assertBusy(() -> assertNotNull(handler.stream.buf()));
 
-            // enable auto-read to receive channel close event
-            handler.stream.channel().config().setAutoRead(true);
             assertFalse(handler.streamClosed);
 
-            // terminate connection and wait resources are released
+            // terminate client connection
             ctx.clientChannel.close();
+            // read the first half of the request
+            handler.stream.next();
+            // attempt to read more data and it should notice channel being closed eventually
+            handler.stream.next();
+
+            // wait for resources to be released
             assertBusy(() -> {
                 assertNull(handler.stream.buf());
                 assertTrue(handler.streamClosed);
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
index b55f4eccafca8..cad839bed9555 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
@@ -103,7 +103,7 @@ public void testDefaultKeepAliveSettings() throws IOException {
             MockTransportService serviceD = buildService("TS_D", VersionInformation.CURRENT, TransportVersion.current(), Settings.EMPTY)
         ) {
 
-            try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), TestProfiles.LIGHT_PROFILE)) {
+            try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalNode(), TestProfiles.LIGHT_PROFILE)) {
                 assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class));
                 Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection();
                 assertThat(conn, instanceOf(TcpTransport.NodeChannels.class));
@@ -147,7 +147,7 @@ public void testTransportProfile() {
             MockTransportService serviceD = buildService("TS_D", VersionInformation.CURRENT, TransportVersion.current(), Settings.EMPTY)
         ) {
 
-            try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), connectionProfile)) {
+            try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalNode(), connectionProfile)) {
                 assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class));
                 Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection();
                 assertThat(conn, instanceOf(TcpTransport.NodeChannels.class));
diff --git a/muted-tests.yml b/muted-tests.yml
index 5d14cabdd46ce..084bf27d6a11b 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -17,27 +17,12 @@ tests:
 - class: "org.elasticsearch.xpack.deprecation.DeprecationHttpIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108628"
   method: "testDeprecatedSettingsReturnWarnings"
-- class: org.elasticsearch.index.store.FsDirectoryFactoryTests
-  method: testStoreDirectory
-  issue: https://github.com/elastic/elasticsearch/issues/110210
-- class: org.elasticsearch.index.store.FsDirectoryFactoryTests
-  method: testPreload
-  issue: https://github.com/elastic/elasticsearch/issues/110211
-- class: org.elasticsearch.backwards.SearchWithMinCompatibleSearchNodeIT
-  method: testMinVersionAsNewVersion
-  issue: https://github.com/elastic/elasticsearch/issues/95384
-- class: org.elasticsearch.backwards.SearchWithMinCompatibleSearchNodeIT
-  method: testCcsMinimizeRoundtripsIsFalse
-  issue: https://github.com/elastic/elasticsearch/issues/101974
 - class: "org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests"
   issue: "https://github.com/elastic/elasticsearch/issues/110408"
   method: "testCreateAndRestorePartialSearchableSnapshot"
 - class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests
   method: testPopulationOfCacheWhenLoadingPrivilegesForAllApplications
   issue: https://github.com/elastic/elasticsearch/issues/110789
-- class: org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFileTests
-  method: testCacheFileCreatedAsSparseFile
-  issue: https://github.com/elastic/elasticsearch/issues/110801
 - class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests
   method: testSystemPropertyDisabled
   issue: https://github.com/elastic/elasticsearch/issues/110949
@@ -68,9 +53,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/111448
 - class: org.elasticsearch.search.SearchServiceTests
   issue: https://github.com/elastic/elasticsearch/issues/111529
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=rollup/security_tests/Index-based access}
-  issue: https://github.com/elastic/elasticsearch/issues/111631
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testSnapshotRestore {cluster=OLD}
   issue: https://github.com/elastic/elasticsearch/issues/111777
@@ -83,12 +65,6 @@ tests:
 - class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
   method: testSnapshotRestore {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/111799
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/80_random_rerank_retriever/Random rerank retriever predictably shuffles results}
-  issue: https://github.com/elastic/elasticsearch/issues/111999
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJobAfterMissingIndex
-  issue: https://github.com/elastic/elasticsearch/issues/112088
 - class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/112147
 - class: org.elasticsearch.smoketest.WatcherYamlRestIT
@@ -97,21 +73,9 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_processor/Test create processor with missing mandatory fields}
   issue: https://github.com/elastic/elasticsearch/issues/112191
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJobAsync
-  issue: https://github.com/elastic/elasticsearch/issues/112212
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/rest-api/watcher/put-watch/line_120}
   issue: https://github.com/elastic/elasticsearch/issues/99517
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testMultiIndexDelete
-  issue: https://github.com/elastic/elasticsearch/issues/112381
-- class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests
-  method: "testAggregateIntermediate {TestCase=<geo_point> #2}"
-  issue: https://github.com/elastic/elasticsearch/issues/112461
-- class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests
-  method: testAggregateIntermediate {TestCase=<geo_point>}
-  issue: https://github.com/elastic/elasticsearch/issues/112463
 - class: org.elasticsearch.xpack.esql.action.ManyShardsIT
   method: testRejection
   issue: https://github.com/elastic/elasticsearch/issues/112406
@@ -120,9 +84,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/112424
 - class: org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/111497
-- class: org.elasticsearch.smoketest.SmokeTestIngestWithAllDepsClientYamlTestSuiteIT
-  method: test {yaml=ingest/80_ingest_simulate/Test ingest simulate with reroute and mapping validation from templates}
-  issue: https://github.com/elastic/elasticsearch/issues/112575
 - class: org.elasticsearch.xpack.security.authc.kerberos.SimpleKdcLdapServerTests
   method: testClientServiceMutualAuthentication
   issue: https://github.com/elastic/elasticsearch/issues/112529
@@ -132,9 +93,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT
   method: testIndexPatternErrorMessageComparison_ESQL_SearchDSL
   issue: https://github.com/elastic/elasticsearch/issues/112630
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testPutJob_GivenFarequoteConfig
-  issue: https://github.com/elastic/elasticsearch/issues/112382
 - class: org.elasticsearch.packaging.test.PackagesSecurityAutoConfigurationTests
   method: test20SecurityNotAutoConfiguredOnReInstallation
   issue: https://github.com/elastic/elasticsearch/issues/112635
@@ -150,26 +108,11 @@ tests:
 - class: org.elasticsearch.xpack.sql.qa.single_node.JdbcSqlSpecIT
   method: test {case-functions.testUcaseInline3}
   issue: https://github.com/elastic/elasticsearch/issues/112643
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDelete_multipleRequest
-  issue: https://github.com/elastic/elasticsearch/issues/112701
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCreateJobInSharedIndexUpdatesMapping
-  issue: https://github.com/elastic/elasticsearch/issues/112729
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testGetJob_GivenNoSuchJob
-  issue: https://github.com/elastic/elasticsearch/issues/112730
 - class: org.elasticsearch.script.StatsSummaryTests
   method: testEqualsAndHashCode
   issue: https://github.com/elastic/elasticsearch/issues/112439
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJobAfterMissingAliases
-  issue: https://github.com/elastic/elasticsearch/issues/112823
 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT
   issue: https://github.com/elastic/elasticsearch/issues/112889
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCreateJob_WithClashingFieldMappingsFails
-  issue: https://github.com/elastic/elasticsearch/issues/113046
 - class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT
   method: test {case-functions.testUcaseInline1}
   issue: https://github.com/elastic/elasticsearch/issues/112641
@@ -185,18 +128,12 @@ tests:
 - class: org.elasticsearch.action.admin.cluster.node.stats.NodeStatsTests
   method: testChunking
   issue: https://github.com/elastic/elasticsearch/issues/113139
-- class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests
-  method: testResponse
-  issue: https://github.com/elastic/elasticsearch/issues/113148
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test30StartStop
   issue: https://github.com/elastic/elasticsearch/issues/113160
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test33JavaChanged
   issue: https://github.com/elastic/elasticsearch/issues/113177
-- class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests
-  method: testErrorMidStream
-  issue: https://github.com/elastic/elasticsearch/issues/113179
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113054
@@ -209,9 +146,6 @@ tests:
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test80JavaOptsInEnvVar
   issue: https://github.com/elastic/elasticsearch/issues/113219
-- class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests
-  method: "testFold {TestCase=<double> #2}"
-  issue: https://github.com/elastic/elasticsearch/issues/113225
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test81JavaOptsInJvmOptions
   issue: https://github.com/elastic/elasticsearch/issues/113313
@@ -221,9 +155,6 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/ccr/apis/follow/post-resume-follow/line_84}
   issue: https://github.com/elastic/elasticsearch/issues/113343
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testDeleteJob_TimingStatsDocumentIsDeleted
-  issue: https://github.com/elastic/elasticsearch/issues/113370
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=search/500_date_range/from, to, include_lower, include_upper deprecated}
   issue: https://github.com/elastic/elasticsearch/pull/113286
@@ -236,15 +167,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceCrudIT
   method: testSupportedStream
   issue: https://github.com/elastic/elasticsearch/issues/113430
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testOutOfOrderData
-  issue: https://github.com/elastic/elasticsearch/issues/113477
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCreateJobsWithIndexNameOption
-  issue: https://github.com/elastic/elasticsearch/issues/113528
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testCantCreateJobWithSameID
-  issue: https://github.com/elastic/elasticsearch/issues/113581
 - class: org.elasticsearch.integration.KibanaUserRoleIntegTests
   method: testFieldMappings
   issue: https://github.com/elastic/elasticsearch/issues/113592
@@ -257,41 +179,18 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments}
   issue: https://github.com/elastic/elasticsearch/issues/101458
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testGetJobs_GivenMultipleJobs
-  issue: https://github.com/elastic/elasticsearch/issues/113654
-- class: org.elasticsearch.xpack.ml.integration.MlJobIT
-  method: testGetJobs_GivenSingleJob
-  issue: https://github.com/elastic/elasticsearch/issues/113655
-- class: org.elasticsearch.xpack.security.authz.interceptor.SearchRequestCacheDisablingInterceptorTests
-  method: testHasRemoteIndices
-  issue: https://github.com/elastic/elasticsearch/issues/113660
-- class: org.elasticsearch.xpack.security.authz.interceptor.SearchRequestCacheDisablingInterceptorTests
-  method: testRequestCacheWillBeDisabledWhenSearchRemoteIndices
-  issue: https://github.com/elastic/elasticsearch/issues/113659
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {categorize.Categorize ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113721
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113722
-- class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanosTests
-  issue: https://github.com/elastic/elasticsearch/issues/113661
-- class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT
-  method: testDeploymentSurvivesRestart {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/112980
 - class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT
   method: testNonGzippedDatabase
   issue: https://github.com/elastic/elasticsearch/issues/113821
 - class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT
   method: testGzippedDatabase
   issue: https://github.com/elastic/elasticsearch/issues/113752
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=indices.split/40_routing_partition_size/more than 1}
-  issue: https://github.com/elastic/elasticsearch/issues/113841
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=indices.split/40_routing_partition_size/nested}
-  issue: https://github.com/elastic/elasticsearch/issues/113842
 - class: org.elasticsearch.threadpool.SimpleThreadPoolIT
   method: testThreadPoolMetrics
   issue: https://github.com/elastic/elasticsearch/issues/108320
@@ -310,83 +209,76 @@ tests:
 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
   method: testPutE5Small_withPlatformSpecificVariant
   issue: https://github.com/elastic/elasticsearch/issues/113950
-- class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests
-  method: testInfer_StreamRequest_ErrorResponse
-  issue: https://github.com/elastic/elasticsearch/issues/114105
 - class: org.elasticsearch.xpack.inference.InferenceCrudIT
   method: testGet
   issue: https://github.com/elastic/elasticsearch/issues/114135
-- class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests
-  method: "testFold {TestCase=<double> #7}"
-  issue: https://github.com/elastic/elasticsearch/issues/114175
-- class: org.elasticsearch.action.bulk.IncrementalBulkIT
-  method: testMultipleBulkPartsWithBackoff
-  issue: https://github.com/elastic/elasticsearch/issues/114181
-- class: org.elasticsearch.action.bulk.IncrementalBulkIT
-  method: testIncrementalBulkLowWatermarkBackOff
-  issue: https://github.com/elastic/elasticsearch/issues/114182
 - class: org.elasticsearch.xpack.ilm.ExplainLifecycleIT
   method: testStepInfoPreservedOnAutoRetry
   issue: https://github.com/elastic/elasticsearch/issues/114220
-- class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests
-  method: testInfer_StreamRequest
-  issue: https://github.com/elastic/elasticsearch/issues/114232
-- class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests
-  method: testInfer_StreamRequest_ErrorResponse
-  issue: https://github.com/elastic/elasticsearch/issues/114327
-- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT
-  method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse}
-  issue: https://github.com/elastic/elasticsearch/issues/114331
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
-  method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search}
-  issue: https://github.com/elastic/elasticsearch/issues/114371
-- class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests
-  method: testInfer_StreamRequest
-  issue: https://github.com/elastic/elasticsearch/issues/114385
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/114412
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/114376
-- class: org.elasticsearch.search.retriever.RankDocsRetrieverBuilderTests
-  method: testRewrite
-  issue: https://github.com/elastic/elasticsearch/issues/114467
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test022InstallPluginsFromLocalArchive
   issue: https://github.com/elastic/elasticsearch/issues/111063
-- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
-  method: test {yaml=reference/esql/esql-across-clusters/line_196}
-  issue: https://github.com/elastic/elasticsearch/issues/114488
 - class: org.elasticsearch.gradle.internal.PublishPluginFuncTest
   issue: https://github.com/elastic/elasticsearch/issues/114492
 - class: org.elasticsearch.xpack.inference.DefaultElserIT
   method: testInferCreatesDefaultElser
   issue: https://github.com/elastic/elasticsearch/issues/114503
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/60_synonym_rule_get/Synonym set not found}
-  issue: https://github.com/elastic/elasticsearch/issues/114432
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/60_synonym_rule_get/Get a synonym rule}
-  issue: https://github.com/elastic/elasticsearch/issues/114443
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/60_synonym_rule_get/Synonym rule not found}
-  issue: https://github.com/elastic/elasticsearch/issues/114444
-- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
-  method: testTermsAggregation
-  issue: https://github.com/elastic/elasticsearch/issues/114554
-- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
-  method: testTermsQuery
-  issue: https://github.com/elastic/elasticsearch/issues/114563
-- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
-  method: testMatchAllQuery
-  issue: https://github.com/elastic/elasticsearch/issues/114607
-- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
-  method: testPushSpatialIntersectsEvalToSource {default}
-  issue: https://github.com/elastic/elasticsearch/issues/114627
-- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
-  method: testPushWhereEvalToSource {default}
-  issue: https://github.com/elastic/elasticsearch/issues/114628
+- class: org.elasticsearch.xpack.inference.integration.ModelRegistryIT
+  method: testGetModel
+  issue: https://github.com/elastic/elasticsearch/issues/114657
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/usage/line_38}
+  issue: https://github.com/elastic/elasticsearch/issues/113694
+- class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
+  method: testEveryActionIsEitherOperatorOnlyOrNonOperator
+  issue: https://github.com/elastic/elasticsearch/issues/102992
+- class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests
+  method: testNoStream
+  issue: https://github.com/elastic/elasticsearch/issues/114788
+- class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT
+  method: testTracingCrossCluster
+  issue: https://github.com/elastic/elasticsearch/issues/112731
+- class: org.elasticsearch.license.LicensingTests
+  issue: https://github.com/elastic/elasticsearch/issues/114865
+- class: org.elasticsearch.packaging.test.EnrollmentProcessTests
+  method: test20DockerAutoFormCluster
+  issue: https://github.com/elastic/elasticsearch/issues/114885
+- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
+  method: testInferDeploysDefaultElser
+  issue: https://github.com/elastic/elasticsearch/issues/114913
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry)}
+  issue: https://github.com/elastic/elasticsearch/issues/115231
+- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
+  method: testInferDeploysDefaultE5
+  issue: https://github.com/elastic/elasticsearch/issues/115361
+- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
+  method: testProcessFileChanges
+  issue: https://github.com/elastic/elasticsearch/issues/115280
+- class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
+  method: testFileSettingsReprocessedOnRestartWithoutVersionChange
+  issue: https://github.com/elastic/elasticsearch/issues/115450
+- class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT
+  method: testDeploymentSurvivesRestart {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/115528
+- class: org.elasticsearch.test.apmintegration.MetricsApmIT
+  method: testApmIntegration
+  issue: https://github.com/elastic/elasticsearch/issues/115415
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/esql/esql-across-clusters/line_197}
+  issue: https://github.com/elastic/elasticsearch/issues/115575
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search}
+  issue: https://github.com/elastic/elasticsearch/issues/115600
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=indices.create/10_basic/Create lookup index}
+  issue: https://github.com/elastic/elasticsearch/issues/115605
 
 # Examples:
 #
diff --git a/plugins/examples/settings.gradle b/plugins/examples/settings.gradle
index 78248ecab92d2..1f168525d4b1d 100644
--- a/plugins/examples/settings.gradle
+++ b/plugins/examples/settings.gradle
@@ -8,7 +8,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "3.17.4"
+  id "com.gradle.develocity" version "3.18.1"
 }
 
 // Include all subdirectories as example projects
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
index 709d6892788c4..c12849d545b33 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TextParams;
@@ -91,15 +92,10 @@ public static class Builder extends FieldMapper.Builder {
 
         private final IndexVersion indexCreatedVersion;
         private final TextParams.Analyzers analyzers;
-        private final boolean isSyntheticSourceEnabledViaIndexMode;
+        private final boolean isSyntheticSourceEnabled;
         private final Parameter<Boolean> store;
 
-        public Builder(
-            String name,
-            IndexVersion indexCreatedVersion,
-            IndexAnalyzers indexAnalyzers,
-            boolean isSyntheticSourceEnabledViaIndexMode
-        ) {
+        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
             this.analyzers = new TextParams.Analyzers(
@@ -108,10 +104,10 @@ public Builder(
                 m -> builder(m).analyzers.positionIncrementGap.getValue(),
                 indexCreatedVersion
             );
-            this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode;
+            this.isSyntheticSourceEnabled = isSyntheticSourceEnabled;
             this.store = Parameter.storeParam(
                 m -> builder(m).store.getValue(),
-                () -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false
+                () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false
             );
         }
 
@@ -172,7 +168,7 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
     }
 
     public static TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled())
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings()))
     );
 
     /**
@@ -560,12 +556,8 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(
-            leafName(),
-            builder.indexCreatedVersion,
-            builder.analyzers.indexAnalyzers,
-            builder.isSyntheticSourceEnabledViaIndexMode
-        ).init(this);
+        return new Builder(leafName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers, builder.isSyntheticSourceEnabled)
+            .init(this);
     }
 
     @Override
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
index 593d4b41df712..6c77186089644 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
@@ -242,7 +242,7 @@ public void testIndexedTermVectors() throws IOException {
 
         withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), reader -> {
             LeafReader leaf = reader.leaves().get(0).reader();
-            Terms terms = leaf.getTermVector(0, "field");
+            Terms terms = leaf.termVectors().get(0, "field");
             TermsEnum iterator = terms.iterator();
             BytesRef term;
             Set<String> foundTerms = new HashSet<>();
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
index 61abd64e98a96..d4c4ccfaa442d 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
@@ -130,7 +130,7 @@ private void assertHighlightOneDoc(
                 }
 
                 TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 1, Sort.INDEXORDER);
-                assertThat(topDocs.totalHits.value, equalTo(1L));
+                assertThat(topDocs.totalHits.value(), equalTo(1L));
                 String rawValue = Strings.collectionToDelimitedString(plainTextForHighlighter, String.valueOf(MULTIVAL_SEP_CHAR));
                 UnifiedHighlighter.Builder builder = UnifiedHighlighter.builder(searcher, hiliteAnalyzer);
                 builder.withBreakIterator(() -> breakIterator);
diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java
index 4594e8d71c6fb..b9f4943b1dab6 100644
--- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java
+++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java
@@ -27,7 +27,6 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
         return new SmbDirectoryWrapper(
             setPreload(
                 new MMapDirectory(location, lockFactory),
-                lockFactory,
                 new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))
             )
         );
diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
index 8ce1bfdc61f6b..3a24427df24a3 100644
--- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
+++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
@@ -89,7 +89,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("xpack.security.enabled", "false")
         // geohex_grid requires gold license
         .setting("xpack.license.self_generated.type", "trial")
-        .feature(FeatureFlag.TIME_SERIES_MODE);
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED);
 
     private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()
         .name(REMOTE_CLUSTER_NAME)
diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
index acdd540ca7b9d..5ada1e941266a 100644
--- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
+++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
@@ -91,6 +91,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("xpack.security.remote_cluster_server.ssl.enabled", "false")
         .setting("xpack.security.remote_cluster_client.ssl.enabled", "false")
         .feature(FeatureFlag.TIME_SERIES_MODE)
+        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .user("test_admin", "x-pack-test-password");
 
     private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local()
diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle
index 6d94e3756eae4..e013ccaf9341e 100644
--- a/qa/ccs-unavailable-clusters/build.gradle
+++ b/qa/ccs-unavailable-clusters/build.gradle
@@ -6,9 +6,12 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-apply plugin: 'elasticsearch.legacy-java-rest-test'
 
-testClusters.matching { it.name == "javaRestTest" }.configureEach {
-  setting 'xpack.security.enabled', 'true'
-  user username: 'admin', password: 'admin-password', role: 'superuser'
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+apply plugin: 'elasticsearch.internal-java-rest-test'
+
+
+tasks.withType(StandaloneRestIntegTestTask).configureEach  {
+  usesDefaultDistribution()
 }
diff --git a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
index 13a77e158c343..780f3994ce627 100644
--- a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
+++ b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -45,6 +46,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -61,6 +63,14 @@ public class CrossClusterSearchUnavailableClusterIT extends ESRestTestCase {
 
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     @Override
     public void tearDown() throws Exception {
         super.tearDown();
@@ -144,7 +154,7 @@ public void testSearchSkipUnavailable() throws IOException {
                 threadPool
             )
         ) {
-            DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
+            DiscoveryNode remoteNode = remoteTransport.getLocalNode();
 
             updateRemoteClusterSettings(Collections.singletonMap("seeds", remoteNode.getAddress().toString()));
 
@@ -297,7 +307,7 @@ public void testSkipUnavailableDependsOnSeeds() throws IOException {
                 threadPool
             )
         ) {
-            DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
+            DiscoveryNode remoteNode = remoteTransport.getLocalNode();
 
             {
                 // check that skip_unavailable alone cannot be set
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 8570662f7b523..92a704f793dc2 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -27,12 +27,12 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
+import org.elasticsearch.search.SearchFeatures;
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
@@ -1202,15 +1202,8 @@ public void testClosedIndices() throws Exception {
             closeIndex(index);
         }
 
-        @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_INDEXING) // This check can be removed (always assume true)
-        var originalClusterSupportsReplicationOfClosedIndices = oldClusterHasFeature(RestTestLegacyFeatures.REPLICATION_OF_CLOSED_INDICES);
-
-        if (originalClusterSupportsReplicationOfClosedIndices) {
-            ensureGreenLongWait(index);
-            assertClosedIndex(index, true);
-        } else {
-            assertClosedIndex(index, false);
-        }
+        ensureGreenLongWait(index);
+        assertClosedIndex(index, true);
 
         if (isRunningAgainstOldCluster() == false) {
             openIndex(index);
@@ -1694,6 +1687,211 @@ public void testSystemIndexMetadataIsUpgraded() throws Exception {
         }
     }
 
+    /**
+     * This test ensures that search results on old indices using "persian" analyzer don't change
+     * after we introduce Lucene 10
+     */
+    public void testPersianAnalyzerBWC() throws Exception {
+        var originalClusterLegacyPersianAnalyzer = oldClusterHasFeature(SearchFeatures.LUCENE_10_0_0_UPGRADE) == false;
+        assumeTrue("Don't run this test if both versions already support stemming", originalClusterLegacyPersianAnalyzer);
+        final String indexName = "test_persian_stemmer";
+        Settings idxSettings = indexSettings(1, 1).build();
+        String mapping = """
+                {
+                  "properties": {
+                    "textfield" : {
+                      "type": "text",
+                      "analyzer": "persian"
+                    }
+                  }
+                }
+            """;
+
+        String query = """
+                {
+                  "query": {
+                    "match": {
+                      "textfield": "كتابها"
+                    }
+                  }
+                }
+            """;
+
+        if (isRunningAgainstOldCluster()) {
+            createIndex(client(), indexName, idxSettings, mapping);
+            ensureGreen(indexName);
+
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + indexName + "/" + "_doc/1",
+                        (builder, params) -> builder.field("textfield", "كتابها")
+                    )
+                )
+            );
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + indexName + "/" + "_doc/2",
+                        (builder, params) -> builder.field("textfield", "كتاب")
+                    )
+                )
+            );
+            refresh(indexName);
+
+            assertNumHits(indexName, 2, 1);
+
+            Request searchRequest = new Request("POST", "/" + indexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(1, entityAsMap(client().performRequest(searchRequest)));
+        } else {
+            // old index should still only return one doc
+            Request searchRequest = new Request("POST", "/" + indexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(1, entityAsMap(client().performRequest(searchRequest)));
+
+            String newIndexName = indexName + "_new";
+            createIndex(client(), newIndexName, idxSettings, mapping);
+            ensureGreen(newIndexName);
+
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + newIndexName + "/" + "_doc/1",
+                        (builder, params) -> builder.field("textfield", "كتابها")
+                    )
+                )
+            );
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + newIndexName + "/" + "_doc/2",
+                        (builder, params) -> builder.field("textfield", "كتاب")
+                    )
+                )
+            );
+            refresh(newIndexName);
+
+            searchRequest = new Request("POST", "/" + newIndexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(2, entityAsMap(client().performRequest(searchRequest)));
+
+            // searching both indices (old and new analysis version) we should get 1 hit from the old and 2 from the new index
+            searchRequest = new Request("POST", "/" + indexName + "," + newIndexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(3, entityAsMap(client().performRequest(searchRequest)));
+        }
+    }
+
+    /**
+     * This test ensures that search results on old indices using "romanain" analyzer don't change
+     * after we introduce Lucene 10
+     */
+    public void testRomanianAnalyzerBWC() throws Exception {
+        var originalClusterLegacyRomanianAnalyzer = oldClusterHasFeature(SearchFeatures.LUCENE_10_0_0_UPGRADE) == false;
+        assumeTrue("Don't run this test if both versions already support stemming", originalClusterLegacyRomanianAnalyzer);
+        final String indexName = "test_romanian_stemmer";
+        Settings idxSettings = indexSettings(1, 1).build();
+        String cedillaForm = "absenţa";
+        String commaForm = "absența";
+
+        String mapping = """
+                {
+                  "properties": {
+                    "textfield" : {
+                      "type": "text",
+                      "analyzer": "romanian"
+                    }
+                  }
+                }
+            """;
+
+        // query that uses the cedilla form of "t"
+        String query = """
+                {
+                  "query": {
+                    "match": {
+                      "textfield": "absenţa"
+                    }
+                  }
+                }
+            """;
+
+        if (isRunningAgainstOldCluster()) {
+            createIndex(client(), indexName, idxSettings, mapping);
+            ensureGreen(indexName);
+
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + indexName + "/" + "_doc/1",
+                        (builder, params) -> builder.field("textfield", cedillaForm)
+                    )
+                )
+            );
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + indexName + "/" + "_doc/2",
+                        // this doc uses the comma form
+                        (builder, params) -> builder.field("textfield", commaForm)
+                    )
+                )
+            );
+            refresh(indexName);
+
+            assertNumHits(indexName, 2, 1);
+
+            Request searchRequest = new Request("POST", "/" + indexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(1, entityAsMap(client().performRequest(searchRequest)));
+        } else {
+            // old index should still only return one doc
+            Request searchRequest = new Request("POST", "/" + indexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(1, entityAsMap(client().performRequest(searchRequest)));
+
+            String newIndexName = indexName + "_new";
+            createIndex(client(), newIndexName, idxSettings, mapping);
+            ensureGreen(newIndexName);
+
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + newIndexName + "/" + "_doc/1",
+                        (builder, params) -> builder.field("textfield", cedillaForm)
+                    )
+                )
+            );
+            assertOK(
+                client().performRequest(
+                    newXContentRequest(
+                        HttpMethod.POST,
+                        "/" + newIndexName + "/" + "_doc/2",
+                        (builder, params) -> builder.field("textfield", commaForm)
+                    )
+                )
+            );
+            refresh(newIndexName);
+
+            searchRequest = new Request("POST", "/" + newIndexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(2, entityAsMap(client().performRequest(searchRequest)));
+
+            // searching both indices (old and new analysis version) we should get 1 hit from the old and 2 from the new index
+            searchRequest = new Request("POST", "/" + indexName + "," + newIndexName + "/_search");
+            searchRequest.setJsonEntity(query);
+            assertTotalHits(3, entityAsMap(client().performRequest(searchRequest)));
+        }
+    }
+
     public void testForbidDisableSoftDeletesOnRestore() throws Exception {
         final String snapshot = "snapshot-" + index;
         if (isRunningAgainstOldCluster()) {
diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
index a5b7ae8d703ea..23d7af7603d56 100644
--- a/qa/mixed-cluster/build.gradle
+++ b/qa/mixed-cluster/build.gradle
@@ -71,6 +71,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
       numberOfNodes = 4
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
       setting 'xpack.security.enabled', 'false'
+      setting "xpack.license.self_generated.type", "trial"
       /* There is a chance we have more master changes than "normal", so to avoid this test from failing,
        we increase the threshold (as this purpose of this test isn't to test that specific indicator). */
       if (bwcVersion.onOrAfter(Version.fromString("8.4.0"))) {
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
deleted file mode 100644
index a391ee5a3bd7b..0000000000000
--- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-package org.elasticsearch.backwards;
-
-import org.apache.http.HttpHost;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.core.Strings;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.test.rest.ObjectPath;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class SearchWithMinCompatibleSearchNodeIT extends ESRestTestCase {
-
-    private static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version");
-    private static final String NEW_NODES_VERSION = System.getProperty("tests.new_nodes_version");
-
-    private static String index = "test_min_version";
-    private static int numShards;
-    private static int numReplicas = 1;
-    private static int numDocs;
-    private static MixedClusterTestNodes nodes;
-    private static List<MixedClusterTestNode> allNodes;
-
-    @Before
-    public void prepareTestData() throws IOException {
-        nodes = MixedClusterTestNodes.buildNodes(client(), BWC_NODES_VERSION);
-        numShards = nodes.size();
-        numDocs = randomIntBetween(numShards, 16);
-        allNodes = new ArrayList<>();
-        allNodes.addAll(nodes.getBWCNodes());
-        allNodes.addAll(nodes.getNewNodes());
-
-        if (client().performRequest(new Request("HEAD", "/" + index)).getStatusLine().getStatusCode() == 404) {
-            createIndex(index, indexSettings(numShards, numReplicas).build());
-            for (int i = 0; i < numDocs; i++) {
-                Request request = new Request("PUT", index + "/_doc/" + i);
-                request.setJsonEntity("{\"test\": \"test_" + randomAlphaOfLength(2) + "\"}");
-                assertOK(client().performRequest(request));
-            }
-            ensureGreen(index);
-        }
-    }
-
-    public void testMinVersionAsNewVersion() throws Exception {
-        try (
-            RestClient client = buildClient(
-                restClientSettings(),
-                allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new)
-            )
-        ) {
-            Request newVersionRequest = new Request(
-                "POST",
-                index + "/_search?min_compatible_shard_node=" + NEW_NODES_VERSION + "&ccs_minimize_roundtrips=false"
-            );
-            assertBusy(() -> {
-                ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(newVersionRequest));
-                assertThat(
-                    responseException.getResponse().getStatusLine().getStatusCode(),
-                    equalTo(RestStatus.INTERNAL_SERVER_ERROR.getStatus())
-                );
-                assertThat(responseException.getMessage(), containsString("""
-                    {"error":{"root_cause":[],"type":"search_phase_execution_exception\""""));
-                assertThat(responseException.getMessage(), containsString(Strings.format("""
-                    caused_by":{"type":"version_mismatch_exception",\
-                    "reason":"One of the shards is incompatible with the required minimum version [%s]\"""", NEW_NODES_VERSION)));
-            });
-        }
-    }
-
-    public void testMinVersionAsOldVersion() throws Exception {
-        try (
-            RestClient client = buildClient(
-                restClientSettings(),
-                allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new)
-            )
-        ) {
-            Request oldVersionRequest = new Request(
-                "POST",
-                index + "/_search?min_compatible_shard_node=" + BWC_NODES_VERSION + "&ccs_minimize_roundtrips=false"
-            );
-            oldVersionRequest.setJsonEntity("""
-                {"query":{"match_all":{}},"_source":false}""");
-            assertBusy(() -> {
-                Response response = client.performRequest(oldVersionRequest);
-                ObjectPath responseObject = ObjectPath.createFromResponse(response);
-                Map<String, Object> shardsResult = responseObject.evaluate("_shards");
-                assertThat(shardsResult.get("total"), equalTo(numShards));
-                assertThat(shardsResult.get("successful"), equalTo(numShards));
-                assertThat(shardsResult.get("failed"), equalTo(0));
-                Map<String, Object> hitsResult = responseObject.evaluate("hits.total");
-                assertThat(hitsResult.get("value"), equalTo(numDocs));
-                assertThat(hitsResult.get("relation"), equalTo("eq"));
-            });
-        }
-    }
-
-    public void testCcsMinimizeRoundtripsIsFalse() throws Exception {
-        try (
-            RestClient client = buildClient(
-                restClientSettings(),
-                allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new)
-            )
-        ) {
-            String version = randomBoolean() ? NEW_NODES_VERSION : BWC_NODES_VERSION;
-
-            Request request = new Request(
-                "POST",
-                index + "/_search?min_compatible_shard_node=" + version + "&ccs_minimize_roundtrips=true"
-            );
-            assertBusy(() -> {
-                ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request));
-                assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
-                assertThat(responseException.getMessage(), containsString("""
-                    {"error":{"root_cause":[{"type":"action_request_validation_exception"\
-                    """));
-                assertThat(
-                    responseException.getMessage(),
-                    containsString(
-                        "\"reason\":\"Validation Failed: 1: "
-                            + "[ccs_minimize_roundtrips] cannot be [true] when setting a minimum compatible shard version;\""
-                    )
-                );
-            });
-        }
-    }
-}
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
index 2a3e0c16fdc2f..8cb8354eb5d71 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
@@ -99,6 +99,7 @@
  *     <li>The default image with a custom, small base image</li>
  *     <li>A UBI-based image</li>
  *     <li>Another UBI image for Iron Bank</li>
+ *     <li>A WOLFI-based image</li>
  *     <li>Images for Cloud</li>
  * </ul>
  */
@@ -168,12 +169,7 @@ public void test012SecurityCanBeDisabled() throws Exception {
      * Checks that no plugins are initially active.
      */
     public void test020PluginsListWithNoPlugins() {
-        assumeTrue(
-            "Only applies to non-Cloud images",
-            distribution.packaging != Packaging.DOCKER_CLOUD
-                && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
-                && distribution().packaging != Packaging.DOCKER_WOLFI_ESS
-        );
+        assumeTrue("Only applies to non-Cloud images", distribution().packaging != Packaging.DOCKER_CLOUD_ESS);
 
         final Installation.Executables bin = installation.executables();
         final Result r = sh.run(bin.pluginTool + " list");
@@ -203,15 +199,14 @@ public void test021InstallPlugin() {
      * Checks that ESS images can install plugins from the local archive.
      */
     public void test022InstallPluginsFromLocalArchive() {
-        assumeTrue(
-            "Only ESS images have a local archive",
-            distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS
-        );
+        assumeTrue("Only ESS images have a local archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS);
 
         final String plugin = "analysis-icu";
         final Installation.Executables bin = installation.executables();
 
+        listPluginArchive().forEach(System.out::println);
         assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin)));
+        assertThat("Expected " + plugin + " available in archive", listPluginArchive(), hasSize(16));
 
         // Stuff the proxy settings with garbage, so any attempt to go out to the internet would fail
         sh.getEnv()
@@ -259,10 +254,7 @@ public void test023InstallPluginUsingConfigFile() {
      * Checks that ESS images can manage plugins from the local archive by deploying a plugins config file.
      */
     public void test024InstallPluginFromArchiveUsingConfigFile() {
-        assumeTrue(
-            "Only ESS image has a plugin archive",
-            distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS
-        );
+        assumeTrue("Only ESS image has a plugin archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS);
 
         final String filename = "elasticsearch-plugins.yml";
         append(tempDir.resolve(filename), """
@@ -394,7 +386,7 @@ public void test040JavaUsesTheOsProvidedKeystore() {
         if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts"));
-        } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_WOLFI_ESS) {
+        } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/ssl/certs/java/cacerts"));
         } else {
@@ -1121,10 +1113,8 @@ public void test170DefaultShellIsBash() {
      */
     public void test171AdditionalCliOptionsAreForwarded() throws Exception {
         assumeTrue(
-            "Does not apply to Cloud and wolfi ess images, because they don't use the default entrypoint",
-            distribution.packaging != Packaging.DOCKER_CLOUD
-                && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
-                && distribution().packaging != Packaging.DOCKER_WOLFI_ESS
+            "Does not apply to Cloud ESS images, because they don't use the default entrypoint",
+            distribution().packaging != Packaging.DOCKER_CLOUD_ESS
         );
 
         runContainer(distribution(), builder().runArgs("bin/elasticsearch", "-Ecluster.name=kimchy").envVar("ELASTIC_PASSWORD", PASSWORD));
@@ -1211,11 +1201,7 @@ public void test310IronBankImageHasNoAdditionalLabels() throws Exception {
      * Check that the Cloud image contains the required Beats
      */
     public void test400CloudImageBundlesBeats() {
-        assumeTrue(
-            distribution.packaging == Packaging.DOCKER_CLOUD
-                || distribution.packaging == Packaging.DOCKER_CLOUD_ESS
-                || distribution.packaging == Packaging.DOCKER_WOLFI_ESS
-        );
+        assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD_ESS);
 
         final List<String> contents = listContents("/opt");
         assertThat("Expected beats in /opt", contents, hasItems("filebeat", "metricbeat"));
@@ -1233,6 +1219,10 @@ private List<String> listPlugins() {
         return sh.run(bin.pluginTool + " list").stdout().lines().collect(Collectors.toList());
     }
 
+    private List<String> listPluginArchive() {
+        return sh.run("ls -lh /opt/plugins/archive").stdout().lines().collect(Collectors.toList());
+    }
+
     /**
      * Check that readiness listener works
      */
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java
index 2aff1f258ed65..02e1ce35764cf 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java
@@ -436,10 +436,7 @@ private void verifyKeystorePermissions() {
         switch (distribution.packaging) {
             case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660));
             case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660));
-            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> assertThat(
-                keystore,
-                DockerFileMatcher.file(p660)
-            );
+            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(keystore, DockerFileMatcher.file(p660));
             default -> throw new IllegalStateException("Unknown Elasticsearch packaging type.");
         }
     }
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
index 487a00bdecac9..b4a00ca56924a 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
@@ -245,7 +245,7 @@ protected static void install() throws Exception {
                 installation = Packages.installPackage(sh, distribution);
                 Packages.verifyPackageInstallation(installation, distribution, sh);
             }
-            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> {
+            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> {
                 installation = Docker.runContainer(distribution);
                 Docker.verifyContainerInstallation(installation);
             }
@@ -335,10 +335,8 @@ public Shell.Result runElasticsearchStartCommand(String password, boolean daemon
             case DOCKER:
             case DOCKER_UBI:
             case DOCKER_IRON_BANK:
-            case DOCKER_CLOUD:
             case DOCKER_CLOUD_ESS:
             case DOCKER_WOLFI:
-            case DOCKER_WOLFI_ESS:
                 // nothing, "installing" docker image is running it
                 return Shell.NO_OP;
             default:
@@ -359,10 +357,8 @@ public void stopElasticsearch() throws Exception {
             case DOCKER:
             case DOCKER_UBI:
             case DOCKER_IRON_BANK:
-            case DOCKER_CLOUD:
             case DOCKER_CLOUD_ESS:
             case DOCKER_WOLFI:
-            case DOCKER_WOLFI_ESS:
                 // nothing, "installing" docker image is running it
                 break;
             default:
@@ -375,8 +371,7 @@ public void awaitElasticsearchStartup(Shell.Result result) throws Exception {
         switch (distribution.packaging) {
             case TAR, ZIP -> Archives.assertElasticsearchStarted(installation);
             case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation);
-            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> Docker
-                .waitForElasticsearchToStart();
+            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart();
             default -> throw new IllegalStateException("Unknown Elasticsearch packaging type.");
         }
     }
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java
index d63d956dc5199..11b8324384631 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java
@@ -33,14 +33,10 @@ public Distribution(Path path) {
             this.packaging = Packaging.DOCKER_UBI;
         } else if (filename.endsWith(".ironbank.tar")) {
             this.packaging = Packaging.DOCKER_IRON_BANK;
-        } else if (filename.endsWith(".cloud.tar")) {
-            this.packaging = Packaging.DOCKER_CLOUD;
         } else if (filename.endsWith(".cloud-ess.tar")) {
             this.packaging = Packaging.DOCKER_CLOUD_ESS;
         } else if (filename.endsWith(".wolfi.tar")) {
             this.packaging = Packaging.DOCKER_WOLFI;
-        } else if (filename.endsWith(".wolfi-ess.tar")) {
-            this.packaging = Packaging.DOCKER_WOLFI_ESS;
         } else {
             int lastDot = filename.lastIndexOf('.');
             this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT));
@@ -65,7 +61,7 @@ public boolean isPackage() {
      */
     public boolean isDocker() {
         return switch (packaging) {
-            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> true;
+            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true;
             default -> false;
         };
     }
@@ -79,10 +75,8 @@ public enum Packaging {
         DOCKER(".docker.tar", Platforms.isDocker()),
         DOCKER_UBI(".ubi.tar", Platforms.isDocker()),
         DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()),
-        DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()),
         DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()),
-        DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()),
-        DOCKER_WOLFI_ESS(".wolfi-ess.tar", Platforms.isDocker());
+        DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker());
 
         /** The extension of this distribution's file */
         public final String extension;
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
index 6f7827663d46c..0cd2823080b9b 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
@@ -532,9 +532,7 @@ public static void verifyContainerInstallation(Installation es) {
                 )
             );
 
-        if (es.distribution.packaging == Packaging.DOCKER_CLOUD
-            || es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS
-            || es.distribution.packaging == Packaging.DOCKER_WOLFI_ESS) {
+        if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
             verifyCloudContainerInstallation(es);
         }
     }
@@ -543,7 +541,7 @@ private static void verifyCloudContainerInstallation(Installation es) {
         final String pluginArchive = "/opt/plugins/archive";
         final List<String> plugins = listContents(pluginArchive);
 
-        if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS || es.distribution.packaging == Packaging.DOCKER_WOLFI_ESS) {
+        if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
             assertThat("ESS image should come with plugins in " + pluginArchive, plugins, not(empty()));
 
             final List<String> repositoryPlugins = plugins.stream()
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java
index a1529de825804..e3eac23d3ecce 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java
@@ -165,10 +165,8 @@ public static String getImageName(Distribution distribution) {
             case DOCKER -> "";
             case DOCKER_UBI -> "-ubi";
             case DOCKER_IRON_BANK -> "-ironbank";
-            case DOCKER_CLOUD -> "-cloud";
             case DOCKER_CLOUD_ESS -> "-cloud-ess";
             case DOCKER_WOLFI -> "-wolfi";
-            case DOCKER_WOLFI_ESS -> "-wolfi-ess";
             default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging);
         };
 
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
index e0d1e7aafa637..17618d5439d48 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
@@ -11,7 +11,6 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
@@ -82,8 +81,7 @@ private void assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent() throws
                     Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
                     1238.49922909,
                     ByteSizeValue.ofGb(32),
-                    ByteSizeValue.ofGb(128),
-                    clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
+                    ByteSizeValue.ofGb(128)
                 )
             )
             .toList();
@@ -153,8 +151,7 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve
                         Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
                         processorsPrecision == ProcessorsPrecision.DOUBLE ? randomDoubleProcessorCount() : 0.5f,
                         ByteSizeValue.ofGb(randomIntBetween(10, 24)),
-                        ByteSizeValue.ofGb(randomIntBetween(128, 256)),
-                        clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
+                        ByteSizeValue.ofGb(randomIntBetween(128, 256))
                     )
                 )
                 .toList();
@@ -167,8 +164,7 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve
                     Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
                     new DesiredNode.ProcessorsRange(minProcessors, minProcessors + randomIntBetween(10, 20)),
                     ByteSizeValue.ofGb(randomIntBetween(10, 24)),
-                    ByteSizeValue.ofGb(randomIntBetween(128, 256)),
-                    clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
+                    ByteSizeValue.ofGb(randomIntBetween(128, 256))
                 );
             }).toList();
         }
@@ -182,8 +178,7 @@ private void addClusterNodesToDesiredNodesWithIntegerProcessors(int version) thr
                     Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
                     randomIntBetween(1, 24),
                     ByteSizeValue.ofGb(randomIntBetween(10, 24)),
-                    ByteSizeValue.ofGb(randomIntBetween(128, 256)),
-                    clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
+                    ByteSizeValue.ofGb(randomIntBetween(128, 256))
                 )
             )
             .toList();
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java
new file mode 100644
index 0000000000000..834d97f755dfb
--- /dev/null
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+public class FileSettingsRoleMappingUpgradeIT extends ParameterizedRollingUpgradeTestCase {
+
+    private static final String settingsJSON = """
+        {
+             "metadata": {
+                 "version": "1",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "everyone_kibana": {
+                          "enabled": true,
+                          "roles": [ "kibana_user" ],
+                          "rules": { "field": { "username": "*" } }
+                       }
+                 }
+             }
+        }""";
+
+    private static final TemporaryFolder repoDirectory = new TemporaryFolder();
+
+    private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(NODE_NUM)
+        .setting("path.repo", new Supplier<>() {
+            @Override
+            @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File")
+            public String get() {
+                return repoDirectory.getRoot().getPath();
+            }
+        })
+        .setting("xpack.security.enabled", "true")
+        // workaround to avoid having to set up clients and authorization headers
+        .setting("xpack.security.authc.anonymous.roles", "superuser")
+        .configFile("operator/settings.json", Resource.fromString(settingsJSON))
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster);
+
+    public FileSettingsRoleMappingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
+    @Before
+    public void checkVersions() {
+        assumeTrue(
+            "Only relevant when upgrading from a version before role mappings were stored in cluster state",
+            oldClusterHasFeature("gte_v8.4.0") && oldClusterHasFeature("gte_v8.15.0") == false
+        );
+    }
+
+    public void testRoleMappingsAppliedOnUpgrade() throws IOException {
+        if (isOldCluster()) {
+            Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
+            List<Object> roleMappings = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get(
+                "metadata.role_mappings.role_mappings"
+            );
+            assertThat(roleMappings, is(nullValue()));
+        } else if (isUpgradedCluster()) {
+            // the nodes have all been upgraded. Check they re-processed the role mappings in the settings file on
+            // upgrade
+            Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
+            List<Object> clusterStateRoleMappings = new XContentTestUtils.JsonMapView(
+                entityAsMap(client().performRequest(clusterStateRequest))
+            ).get("metadata.role_mappings.role_mappings");
+            assertThat(clusterStateRoleMappings, is(not(nullValue())));
+            assertThat(clusterStateRoleMappings.size(), equalTo(1));
+
+            assertThat(
+                entityAsMap(client().performRequest(new Request("GET", "/_security/role_mapping"))).keySet(),
+                // TODO change this to `contains` once the clean-up migration work is merged
+                hasItem("everyone_kibana-read-only-operator-mapping")
+            );
+        }
+    }
+}
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java
index e7cff5cca5a92..a20981a119d8f 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java
@@ -55,14 +55,13 @@ public static Iterable<Object[]> parameters() {
     protected abstract ElasticsearchCluster getUpgradeCluster();
 
     @Before
-    public void extractOldClusterFeatures() {
+    public void upgradeNode() throws Exception {
+        // extract old cluster features
         if (isOldCluster() && oldClusterTestFeatureService == null) {
             oldClusterTestFeatureService = testFeatureService;
         }
-    }
 
-    @Before
-    public void extractOldIndexVersion() throws Exception {
+        // extract old index version
         if (oldIndexVersion == null && upgradedNodes.isEmpty()) {
             IndexVersion indexVersion = null;   // these should all be the same version
 
@@ -93,13 +92,11 @@ public void extractOldIndexVersion() throws Exception {
             assertThat("Index version could not be read", indexVersion, notNullValue());
             oldIndexVersion = indexVersion;
         }
-    }
 
-    @Before
-    public void upgradeNode() throws Exception {
         // Skip remaining tests if upgrade failed
         assumeFalse("Cluster upgrade failed", upgradeFailed);
 
+        // finally, upgrade node
         if (upgradedNodes.size() < requestedUpgradedNodes) {
             closeClients();
             // we might be running a specific upgrade test by itself - check previous nodes too
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java
similarity index 81%
rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java
rename to qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java
index da05011696274..369d0824bdb28 100644
--- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java
+++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java
@@ -9,6 +9,8 @@
 
 package org.elasticsearch.http;
 
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
 import org.elasticsearch.action.bulk.IncrementalBulkService;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
@@ -19,24 +21,30 @@
 import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0)
-public class IncrementalBulkRestIT extends HttpSmokeTestCase {
+public class BulkRestIT extends HttpSmokeTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         return Settings.builder()
             .put(super.nodeSettings(nodeOrdinal, otherSettings))
-            .put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), true)
+            .put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), seventyFivePercentOfTheTime())
             .build();
     }
 
+    private static boolean seventyFivePercentOfTheTime() {
+        return (randomBoolean() && randomBoolean()) == false;
+    }
+
     public void testBulkUriMatchingDoesNotMatchBulkCapabilitiesApi() throws IOException {
         Request request = new Request("GET", "/_capabilities?method=GET&path=%2F_bulk&capabilities=failure_store_status&pretty");
         Response response = getRestClient().performRequest(request);
@@ -51,6 +59,26 @@ public void testBulkMissingBody() throws IOException {
         assertThat(responseException.getMessage(), containsString("request body is required"));
     }
 
+    public void testBulkInvalidIndexNameString() throws IOException {
+        Request request = new Request("POST", "/_bulk");
+
+        byte[] bytes1 = "{\"create\":{\"_index\":\"".getBytes(StandardCharsets.UTF_8);
+        byte[] bytes2 = new byte[] { (byte) 0xfe, (byte) 0xfe, (byte) 0xff, (byte) 0xff };
+        byte[] bytes3 = "\",\"_id\":\"1\"}}\n{\"field\":1}\n\r\n".getBytes(StandardCharsets.UTF_8);
+        byte[] bulkBody = new byte[bytes1.length + bytes2.length + bytes3.length];
+        System.arraycopy(bytes1, 0, bulkBody, 0, bytes1.length);
+        System.arraycopy(bytes2, 0, bulkBody, bytes1.length, bytes2.length);
+        System.arraycopy(bytes3, 0, bulkBody, bytes1.length + bytes2.length, bytes3.length);
+
+        request.setEntity(new ByteArrayEntity(bulkBody, ContentType.APPLICATION_JSON));
+
+        ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+        assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(BAD_REQUEST.getStatus()));
+        assertThat(responseException.getMessage(), containsString("could not parse bulk request body"));
+        assertThat(responseException.getMessage(), containsString("json_parse_exception"));
+        assertThat(responseException.getMessage(), containsString("Invalid UTF-8"));
+    }
+
     public void testBulkRequestBodyImproperlyTerminated() throws IOException {
         Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk");
         // missing final line of the bulk body. cannot process
@@ -61,10 +89,10 @@ public void testBulkRequestBodyImproperlyTerminated() throws IOException {
         );
         ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
         assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
-        assertThat(responseException.getMessage(), containsString("could not parse bulk request body"));
+        assertThat(responseException.getMessage(), containsString("The bulk request must be terminated by a newline"));
     }
 
-    public void testIncrementalBulk() throws IOException {
+    public void testBulkRequest() throws IOException {
         Request createRequest = new Request("PUT", "/index_name");
         createRequest.setJsonEntity("""
             {
@@ -81,7 +109,6 @@ public void testIncrementalBulk() throws IOException {
 
         Request firstBulkRequest = new Request("POST", "/index_name/_bulk");
 
-        // index documents for the rollup job
         String bulkBody = "{\"index\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n"
             + "{\"field\":1}\n"
             + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"2\"}}\n"
@@ -113,7 +140,6 @@ public void testBulkWithIncrementalDisabled() throws IOException {
 
         Request firstBulkRequest = new Request("POST", "/index_name/_bulk");
 
-        // index documents for the rollup job
         String bulkBody = "{\"index\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n"
             + "{\"field\":1}\n"
             + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"2\"}}\n"
@@ -137,7 +163,7 @@ public void testBulkWithIncrementalDisabled() throws IOException {
         }
     }
 
-    public void testIncrementalMalformed() throws IOException {
+    public void testMalformedActionLineBulk() throws IOException {
         Request createRequest = new Request("PUT", "/index_name");
         createRequest.setJsonEntity("""
             {
@@ -154,7 +180,6 @@ public void testIncrementalMalformed() throws IOException {
 
         Request bulkRequest = new Request("POST", "/index_name/_bulk");
 
-        // index documents for the rollup job
         final StringBuilder bulk = new StringBuilder();
         bulk.append("{\"index\":{\"_index\":\"index_name\"}}\n");
         bulk.append("{\"field\":1}\n");
@@ -170,7 +195,6 @@ public void testIncrementalMalformed() throws IOException {
     private static void sendLargeBulk() throws IOException {
         Request bulkRequest = new Request("POST", "/index_name/_bulk");
 
-        // index documents for the rollup job
         final StringBuilder bulk = new StringBuilder();
         bulk.append("{\"delete\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n");
         int updates = 0;
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml
index a8f7e1e5877c8..cc767dfa56597 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml
@@ -214,7 +214,7 @@
 "Test rolling up json object arrays":
   - do:
       ingest.put_pipeline:
-        id: "_id"
+        id: "pipeline-id"
         body:  >
           {
             "processors": [
@@ -237,7 +237,7 @@
       index:
         index: test
         id: "1"
-        pipeline: "_id"
+        pipeline: "pipeline-id"
         body: {
           values_flat : [],
           values: [
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml
index 9a7444c4ffc6c..ef790843b7bfb 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml
@@ -2,7 +2,7 @@
 "Test with date processor":
   - do:
       ingest.put_pipeline:
-        id: "_id"
+        id: "pipeline-id"
         body:  >
           {
             "processors": [
@@ -44,7 +44,7 @@
       index:
         index: test
         id: "1"
-        pipeline: "_id"
+        pipeline: "pipeline-id"
         body: {
           log: "89.160.20.128 - - [08/Sep/2014:02:54:42 +0000] \"GET /presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png HTTP/1.1\" 200 175208 \"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\" \"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\""
         }
@@ -71,7 +71,7 @@
 "Test with date processor and ECS-v1":
   - do:
       ingest.put_pipeline:
-        id: "_id"
+        id: "pipeline-id"
         body:  >
           {
             "processors": [
@@ -102,7 +102,7 @@
       index:
         index: test
         id: "1"
-        pipeline: "_id"
+        pipeline: "pipeline-id"
         body: {
           log: "89.160.20.128 - - [08/Sep/2014:02:54:42 +0000] \"GET /presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png HTTP/1.1\" 200 175208 \"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\" \"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\""
         }
@@ -128,7 +128,7 @@
 "Test mutate":
   - do:
       ingest.put_pipeline:
-        id: "_id"
+        id: "pipeline-id"
         body:  >
           {
             "processors": [
@@ -188,7 +188,7 @@
       index:
         index: test
         id: "1"
-        pipeline: "_id"
+        pipeline: "pipeline-id"
         body: {
           "age" : 33,
           "eyeColor" : "brown",
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
index 18eb401aaa0fe..2d3fa6b568381 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
@@ -1216,3 +1216,507 @@ setup:
   - match: { docs.0.doc._source.foo: "FOO" }
   - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] }
   - not_exists: docs.0.doc.error
+
+---
+"Test ingest simulate with mapping addition for data streams":
+  # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data
+  # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  - requires:
+      cluster_features: ["simulate.mapping.addition"]
+      reason: "ingest simulate mapping addition added in 8.17"
+
+  - do:
+      headers:
+        Content-Type: application/json
+      ingest.put_pipeline:
+        id: "foo-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "set": {
+                  "field": "foo",
+                  "value": true
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      cluster.put_component_template:
+        name: mappings_template
+        body:
+          template:
+            mappings:
+              dynamic: strict
+              properties:
+                foo:
+                  type: boolean
+
+  - do:
+      cluster.put_component_template:
+        name: settings_template
+        body:
+          template:
+            settings:
+              index:
+                default_pipeline: "foo-pipeline"
+
+  - do:
+      allowed_warnings:
+        - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test-composable-1
+        body:
+          index_patterns:
+            - foo*
+          composed_of:
+            - mappings_template
+            - settings_template
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template-1
+        body:
+          index_patterns: [simple-data-stream1]
+          composed_of:
+            - mappings_template
+            - settings_template
+          data_stream: {}
+
+  # Here we replace my-template-1 with a substitute version that uses the settings_template_2 and mappings_template_2 templates defined in
+  # this request, and foo-pipeline-2 defined in this request.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: simple-data-stream1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "@timestamp": 1234,
+                  "foo": false
+                }
+              }
+            ],
+            "pipeline_substitutions": {
+              "foo-pipeline-2": {
+                "processors": [
+                  {
+                    "set": {
+                      "field": "foo",
+                      "value": "FOO"
+                    }
+                  }
+                ]
+              }
+            },
+            "component_template_substitutions": {
+              "settings_template_2": {
+                "template": {
+                  "settings": {
+                    "index": {
+                      "default_pipeline": "foo-pipeline-2"
+                    }
+                  }
+                }
+              },
+              "mappings_template_2": {
+                "template": {
+                  "mappings": {
+                    "dynamic": "strict",
+                    "properties": {
+                      "foo": {
+                        "type": "integer"
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            "index_template_substitutions": {
+              "my-template-1": {
+                "index_patterns": ["simple-data-stream1"],
+                "composed_of": ["settings_template_2", "mappings_template_2"],
+                "data_stream": {}
+              }
+            },
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "foo": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "simple-data-stream1" }
+  - match: { docs.0.doc._source.foo: "FOO" }
+  - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] }
+  - not_exists: docs.0.doc.error
+
+  - do:
+      indices.create_data_stream:
+        name: simple-data-stream1
+  - is_true: acknowledged
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  # Now that we have created a data stream, run the exact same simulate ingeset request to make sure we still get the same result, and that
+  # the substitutions and additions from the simulate ingest request are used instead of information from the data stream or its backing
+  # index.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: simple-data-stream1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "@timestamp": 1234,
+                  "foo": false
+                }
+              }
+            ],
+            "pipeline_substitutions": {
+              "foo-pipeline-2": {
+                "processors": [
+                  {
+                    "set": {
+                      "field": "foo",
+                      "value": "FOO"
+                    }
+                  }
+                ]
+              }
+            },
+            "component_template_substitutions": {
+              "settings_template_2": {
+                "template": {
+                  "settings": {
+                    "index": {
+                      "default_pipeline": "foo-pipeline-2"
+                    }
+                  }
+                }
+              },
+              "mappings_template_2": {
+                "template": {
+                  "mappings": {
+                    "dynamic": "strict",
+                    "properties": {
+                      "foo": {
+                        "type": "integer"
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            "index_template_substitutions": {
+              "my-template-1": {
+                "index_patterns": ["simple-data-stream1"],
+                "composed_of": ["settings_template_2", "mappings_template_2"],
+                "data_stream": {}
+              }
+            },
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "foo": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "simple-data-stream1" }
+  - match: { docs.0.doc._source.foo: "FOO" }
+  - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] }
+  - not_exists: docs.0.doc.error
+
+---
+"Test mapping addition works with legacy templates":
+  # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data
+  # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  - requires:
+      cluster_features: ["simulate.mapping.addition"]
+      reason: "ingest simulate mapping addition added in 8.17"
+
+  - do:
+      indices.put_template:
+        name: my-legacy-template
+        body:
+          index_patterns: foo-*
+          settings:
+            number_of_replicas: 0
+          mappings:
+            dynamic: strict
+            properties:
+              foo:
+                type: integer
+              bar:
+                type: boolean
+
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "not a boolean"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "not a boolean" }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "not a boolean"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "bar": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "not a boolean" }
+  - not_exists: docs.0.doc.error
+
+  - do:
+      allowed_warnings:
+        - "index [foo-1] matches multiple legacy templates [global, my-legacy-template], composable templates will only match a single template"
+      indices.create:
+        index: foo-1
+  - match: { acknowledged: true }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "not a boolean"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "bar": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "not a boolean" }
+  - not_exists: docs.0.doc.error
+
+---
+"Test mapping addition works with indices without templates":
+  # In this test, we make sure that when we have an index that has mapping but was not built with a template, that the additional_mapping
+  # is merged in with that template.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  - requires:
+      cluster_features: ["simulate.support.non.template.mapping"]
+      reason: "ingest simulate support for indices with mappings that didn't come from templates added in 8.17"
+
+  # A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
+  # because this test is making sure we get correct behavior when an index matches *no* template:
+  - do:
+      indices.delete_template:
+        name:   '*'
+        ignore: 404
+
+  # First, make sure that validation fails before we create the index (since we are only defining to bar field but trying to index a value
+  # for foo.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "bar": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" }
+
+  - do:
+      indices.create:
+        index: foo-1
+        body:
+          mappings:
+            dynamic: strict
+            properties:
+              foo:
+                type: integer
+  - match: { acknowledged: true }
+
+  # Now make sure that the mapping for the newly-created index is getting picked up. Validation fails because it only defined a mapping
+  # for foo, not for bar.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" }
+
+  # Now we make sure that the index's mapping gets merged with the mapping_addition:
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "bar": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - not_exists: docs.0.doc.error
+
+  # This last call to simulate is just making sure that if there are no templates, no index mappings, no substitutions, and no mapping
+  # addition, then validation does not fail
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: nonexistent
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "nonexistent" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - not_exists: docs.0.doc.error
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_deprecation_warnings_on_invalid_names_ingest.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_deprecation_warnings_on_invalid_names_ingest.yml
new file mode 100644
index 0000000000000..64f5ccc4609ac
--- /dev/null
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_deprecation_warnings_on_invalid_names_ingest.yml
@@ -0,0 +1,28 @@
+---
+"Test invalid name warnings":
+  - requires:
+      cluster_features: [ "ingest.pipeline_name_special_chars_warning" ]
+      test_runner_features: [ "warnings" ]
+      reason: verifying deprecation warnings from 9.0 onwards for invalid pipeline names
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      ingest.put_pipeline:
+        id: "Invalid*-pipeline:id"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "field" : "field1",
+                  "value": "_value"
+                }
+              }]
+          }
+      warnings:
+        - "Invalid pipeline id: Invalid*-pipeline:id"
+  - match: { acknowledged: true }
diff --git a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
index c68d27b883c53..e53c0564be297 100644
--- a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
@@ -35,6 +35,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         // The first node does not have the ingest role so we're sure ingest requests are forwarded:
         .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]"))
         .feature(FeatureFlag.TIME_SERIES_MODE)
+        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .build();
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
diff --git a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java
index 4b3f623073322..57b0908508bb3 100644
--- a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java
+++ b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java
@@ -150,7 +150,7 @@ public String getName() {
 
         @Override
         public List<Route> routes() {
-            return List.of(Route.builder(Method.PUT, "/_net_new_sys_index/_create").build());
+            return List.of(new Route(Method.PUT, "/_net_new_sys_index/_create"));
         }
 
         @Override
diff --git a/renovate.json b/renovate.json
index 0a1d588e6332c..293a2bb262375 100644
--- a/renovate.json
+++ b/renovate.json
@@ -30,7 +30,7 @@
         "\\s*\"?(?<depName>[^\\s:@\"]+)(?::(?<currentValue>[-a-zA-Z0-9.]+))?(?:@(?<currentDigest>sha256:[a-zA-Z0-9]+))?\"?"
       ],
       "currentValueTemplate": "{{#if currentValue}}{{{currentValue}}}{{else}}latest{{/if}}",
-      "autoReplaceStringTemplate": "\"{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"",
+      "autoReplaceStringTemplate": "{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"",
       "datasourceTemplate": "docker"
     }
   ]
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index a742e83255bbb..1a398f79085e7 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -54,7 +54,13 @@ tasks.named("precommit").configure {
   dependsOn 'enforceYamlTestConvention'
 }
 
-tasks.named("yamlRestCompatTestTransform").configure({task ->
-    task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility")
-    task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility")
+tasks.named("yamlRestCompatTestTransform").configure ({ task ->
+  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling")
+  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count")
+  task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility")
+  task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility")
+  task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "temporary until backported")
+  task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
+  task.skipTest("cluster.desired_nodes/10_basic/Test delete desired nodes with node_version generates a warning", "node_version warning is removed in 9.0")
+  task.skipTest("cluster.desired_nodes/10_basic/Test update desired nodes with node_version generates a warning", "node_version warning is removed in 9.0")
 })
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
index db49daeea372b..d3856b455efd1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
@@ -36,10 +36,6 @@
         "type":"string",
         "description":"a short version of the Accept header, e.g. json, yaml"
       },
-      "local":{
-        "type":"boolean",
-        "description":"Return local information, do not retrieve the state from master node (default: false)"
-      },
       "h":{
         "type":"list",
         "description":"Comma-separated list of column names to display"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json
index a3106a982b809..92b3ce61b4603 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-delete-lifecycle.html",
       "description":"Deletes the data stream lifecycle of the selected data streams."
     },
-    "stability":"experimental",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
index b70854fdc3eb2..7d7a9c96c6419 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
@@ -61,10 +61,6 @@
         ],
         "default":"all",
         "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
-      },
-      "local":{
-        "type":"boolean",
-        "description":"Return local information, do not retrieve the state from master node (default: false)"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json
index 3232407000b19..14e07ee28a80d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json
@@ -4,7 +4,7 @@
       "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams-explain-lifecycle.html",
       "description": "Retrieves information about the index's current data stream lifecycle, such as any potential encountered error, time since creation etc."
     },
-    "stability": "experimental",
+    "stability": "stable",
     "visibility": "public",
     "headers": {
       "accept": [
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
index 0a4e4bb9ed90c..dc02a65adb068 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
@@ -79,10 +79,6 @@
         ],
         "default": "all",
         "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
-      },
-      "local":{
-        "type":"boolean",
-        "description":"Return local information, do not retrieve the state from master node (default: false)"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json
index 6f05af1485f98..a8d2e7185db83 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-get-lifecycle.html",
       "description":"Returns the data stream lifecycle of the selected data streams."
     },
-    "stability":"experimental",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json
index 591525f3d99ff..08dc7128234b9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-put-lifecycle.html",
       "description":"Updates the data stream lifecycle of the selected data streams."
     },
-    "stability":"experimental",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json
new file mode 100644
index 0000000000000..e97d1da276906
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json
@@ -0,0 +1,31 @@
+{
+  "ingest.delete_ip_location_database":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-ip-location-database-api.html",
+      "description":"Deletes an ip location database configuration"
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_ingest/ip_location/database/{id}",
+          "methods":[
+            "DELETE"
+          ],
+          "parts":{
+            "id":{
+              "type":"list",
+              "description":"A comma-separated list of ip location database configurations to delete"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json
new file mode 100644
index 0000000000000..a2e42fe6c8e59
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json
@@ -0,0 +1,37 @@
+{
+  "ingest.get_ip_location_database":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-ip-location-database-api.html",
+      "description":"Returns the specified ip location database configuration"
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_ingest/ip_location/database",
+          "methods":[
+            "GET"
+          ]
+        },
+        {
+          "path":"/_ingest/ip_location/database/{id}",
+          "methods":[
+            "GET"
+          ],
+          "parts":{
+            "id":{
+              "type":"list",
+              "description":"A comma-separated list of ip location database configurations to get; use `*` to get all ip location database configurations"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json
new file mode 100644
index 0000000000000..18487969b1a90
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json
@@ -0,0 +1,36 @@
+{
+  "ingest.put_ip_location_database":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-ip-location-database-api.html",
+      "description":"Puts the configuration for a ip location database to be downloaded"
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_ingest/ip_location/database/{id}",
+          "methods":[
+            "PUT"
+          ],
+          "parts":{
+            "id":{
+              "type":"string",
+              "description":"The id of the database configuration"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+    },
+    "body":{
+      "description":"The database configuration definition",
+      "required":true
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json b/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json
new file mode 100644
index 0000000000000..c82b45771ac7f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json
@@ -0,0 +1,38 @@
+{
+  "query_rules.test": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/test-query-ruleset.html",
+      "description": "Tests a query ruleset to identify the rules that would match input criteria"
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query_rules/{ruleset_id}/_test",
+          "methods": [
+            "POST"
+          ],
+          "parts": {
+            "ruleset_id": {
+              "type": "string",
+              "description": "The unique identifier of the ruleset to test."
+            }
+          }
+        }
+      ]
+    },
+    "body": {
+      "description": "The match criteria to test against the ruleset",
+      "required": true
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index b5dc4d62a2f0f..25b4efd9c4c37 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -237,10 +237,6 @@
         "description":"Indicates whether hits.total should be rendered as an integer or an object in the rest search response",
         "default":false
       },
-      "min_compatible_shard_node":{
-        "type":"string",
-        "description":"The minimum compatible version that all shards involved in search should have for this request to be successful"
-      },
       "include_named_queries_score":{
           "type": "boolean",
           "description":"Indicates whether hit.matched_queries should be rendered as a map that includes the name of the matched query associated with its score (true) or as an array containing the name of the matched queries (false)",
diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
index 2b20e35019424..084e212a913b2 100644
--- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
+++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
@@ -36,6 +36,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .module("health-shards-availability")
         .module("data-streams")
         .feature(FeatureFlag.TIME_SERIES_MODE)
+        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .build();
 
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml
index 2e5234bd1ced1..6118453d7805e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml
@@ -484,16 +484,3 @@
                 test_alias \s+ test_index\n
                 my_alias \s+ test_index\n
             $/
-
----
-"Deprecated local parameter":
-  - requires:
-      cluster_features: ["gte_v8.12.0"]
-      test_runner_features: ["warnings"]
-      reason: verifying deprecation warnings from 8.12.0 onwards
-
-  - do:
-      cat.aliases:
-        local: true
-      warnings:
-        - "the [?local=true] query parameter to cat-aliases requests has no effect and will be removed in a future version"
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml
index 1d1aa524ffb21..a45146a4e147a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml
@@ -59,61 +59,6 @@ teardown:
   - contains: { nodes: { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb" } }
   - contains: { nodes: { settings: { node: { name: "instance-000188" } }, processors: 16.0, memory: "128gb", storage: "1tb" } }
 ---
-"Test update desired nodes with node_version generates a warning":
-  - skip:
-      reason: "contains is a newly added assertion"
-      features: ["contains", "allowed_warnings"]
-  - do:
-      cluster.state: {}
-
-  # Get master node id
-  - set: { master_node: master }
-
-  - do:
-      nodes.info: {}
-  - set: { nodes.$master.version: es_version }
-
-  - do:
-      _internal.update_desired_nodes:
-        history_id: "test"
-        version: 1
-        body:
-            nodes:
-              - { settings: { "node.name": "instance-000187" }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
-      allowed_warnings:
-        - "[version removal] Specifying node_version in desired nodes requests is deprecated."
-  - match: { replaced_existing_history_id: false }
-
-  - do:
-      _internal.get_desired_nodes: {}
-  - match:
-      $body:
-        history_id: "test"
-        version: 1
-        nodes:
-          - { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
-
-  - do:
-      _internal.update_desired_nodes:
-        history_id: "test"
-        version: 2
-        body:
-          nodes:
-            - { settings: { "node.name": "instance-000187" }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
-            - { settings: { "node.name": "instance-000188" }, processors: 16.0, memory: "128gb", storage: "1tb", node_version: $es_version }
-      allowed_warnings:
-        - "[version removal] Specifying node_version in desired nodes requests is deprecated."
-  - match: { replaced_existing_history_id: false }
-
-  - do:
-      _internal.get_desired_nodes: {}
-
-  - match: { history_id: "test" }
-  - match: { version: 2 }
-  - length: { nodes: 2 }
-  - contains: { nodes: { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version } }
-  - contains: { nodes: { settings: { node: { name: "instance-000188" } }, processors: 16.0, memory: "128gb", storage: "1tb", node_version: $es_version } }
----
 "Test update move to a new history id":
   - skip:
       reason: "contains is a newly added assertion"
@@ -199,46 +144,6 @@ teardown:
       _internal.get_desired_nodes: {}
   - match: { status: 404 }
 ---
-"Test delete desired nodes with node_version generates a warning":
-  - skip:
-      features: allowed_warnings
-  - do:
-      cluster.state: {}
-
-  - set: { master_node: master }
-
-  - do:
-      nodes.info: {}
-  - set: { nodes.$master.version: es_version }
-
-  - do:
-      _internal.update_desired_nodes:
-        history_id: "test"
-        version: 1
-        body:
-          nodes:
-            - { settings: { "node.external_id": "instance-000187" }, processors: 8.0, memory: "64gb", storage: "128gb", node_version: $es_version }
-      allowed_warnings:
-        - "[version removal] Specifying node_version in desired nodes requests is deprecated."
-  - match: { replaced_existing_history_id: false }
-
-  - do:
-      _internal.get_desired_nodes: {}
-  - match:
-      $body:
-        history_id: "test"
-        version: 1
-        nodes:
-          - { settings: { node: { external_id: "instance-000187" } }, processors: 8.0, memory: "64gb", storage: "128gb", node_version: $es_version }
-
-  - do:
-      _internal.delete_desired_nodes: {}
-
-  - do:
-      catch: missing
-      _internal.get_desired_nodes: {}
-  - match: { status: 404 }
----
 "Test update desired nodes is idempotent":
   - skip:
       reason: "contains is a newly added assertion"
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml
index 955c68634e617..689c58dad31e6 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml
@@ -121,10 +121,10 @@
   - is_true: ccs._search.total
   - is_true: ccs._search.success
   - exists: ccs._search.skipped
-  - is_true: ccs._search.took
-  - is_true: ccs._search.took.max
-  - is_true: ccs._search.took.avg
-  - is_true: ccs._search.took.p90
+  - exists: ccs._search.took
+  - exists: ccs._search.took.max
+  - exists: ccs._search.took.avg
+  - exists: ccs._search.took.p90
   - is_true: ccs._search.took_mrt_true
   - exists: ccs._search.took_mrt_true.max
   - exists: ccs._search.took_mrt_true.avg
@@ -145,7 +145,7 @@
   - gte: {ccs._search.clusters.cluster_two.total: 1}
   - exists: ccs._search.clusters.cluster_one.skipped
   - exists: ccs._search.clusters.cluster_two.skipped
-  - is_true: ccs._search.clusters.cluster_one.took
-  - is_true: ccs._search.clusters.cluster_one.took.max
-  - is_true: ccs._search.clusters.cluster_one.took.avg
-  - is_true: ccs._search.clusters.cluster_one.took.p90
+  - exists: ccs._search.clusters.cluster_one.took
+  - exists: ccs._search.clusters.cluster_one.took.max
+  - exists: ccs._search.clusters.cluster_one.took.avg
+  - exists: ccs._search.clusters.cluster_one.took.p90
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml
index 8242b7cdd29e7..d0e1759073e1b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml
@@ -149,3 +149,70 @@
       indices.exists_alias:
         name: logs_2022-12-31
   - is_true: ''
+
+---
+"Create lookup index":
+  - requires:
+      test_runner_features: [ capabilities, default_shards ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ lookup_index_mode ]
+      reason: "Support for 'lookup' index mode capability required"
+  - do:
+      indices.create:
+        index: "test_lookup"
+        body:
+          settings:
+            index.mode: lookup
+
+  - do:
+      indices.get_settings:
+        index: test_lookup
+
+  - match: { test_lookup.settings.index.number_of_shards: "1"}
+  - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
+
+---
+"Create lookup index with one shard":
+  - requires:
+      test_runner_features: [ capabilities, default_shards ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ lookup_index_mode ]
+      reason: "Support for 'lookup' index mode capability required"
+  - do:
+      indices.create:
+        index: "test_lookup"
+        body:
+          settings:
+            index:
+              mode: lookup
+              number_of_shards: 1
+
+  - do:
+      indices.get_settings:
+        index: test_lookup
+
+  - match: { test_lookup.settings.index.number_of_shards: "1"}
+  - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
+
+---
+"Create lookup index with two shards":
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ lookup_index_mode ]
+      reason: "Support for 'lookup' index mode capability required"
+  - do:
+      catch: /illegal_argument_exception/
+      indices.create:
+        index: test_lookup
+        body:
+          settings:
+            index.mode: lookup
+            index.number_of_shards: 2
+
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
index dfe6c9820a16a..6a4e92f694220 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
@@ -319,8 +319,8 @@ object param - nested object array next to other fields:
 ---
 object param - nested object with stored array:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
-      reason: requires tracking ignored source
+      cluster_features: ["mapper.ignored_source.always_store_object_arrays_in_nested", "mapper.bwc_workaround_9_0"]
+      reason: requires fix to object array handling
 
   - do:
       indices.create:
@@ -356,8 +356,11 @@ object param - nested object with stored array:
         sort: name
   - match:  { hits.total.value: 2 }
   - match:  { hits.hits.0._source.name: A }
-  - match: { hits.hits.0._source.nested_array_regular.0.b.c: [ 10, 100] }
-  - match: { hits.hits.0._source.nested_array_regular.1.b.c: [ 20, 200] }
+    # due to a workaround for #115261
+  - match: { hits.hits.0._source.nested_array_regular.0.b.0.c: 10 }
+  - match: { hits.hits.0._source.nested_array_regular.0.b.1.c: 100 }
+  - match: { hits.hits.0._source.nested_array_regular.1.b.0.c: 20 }
+  - match: { hits.hits.0._source.nested_array_regular.1.b.1.c: 200 }
   - match: { hits.hits.1._source.name: B }
   - match: { hits.hits.1._source.nested_array_stored.0.b.0.c: 10 }
   - match: { hits.hits.1._source.nested_array_stored.0.b.1.c: 100 }
@@ -411,6 +414,55 @@ index param - nested array within array:
   - match: { hits.hits.0._source.path.to.some.3.id: [ 1000, 2000 ] }
 
 
+---
+index param - nested array within array - disabled second pass:
+  - requires:
+      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      reason: requires tracking ignored source
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              synthetic_source:
+                enable_second_doc_parsing_pass: false
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              name:
+                type: keyword
+              path:
+                properties:
+                  to:
+                    properties:
+                      some:
+                        synthetic_source_keep: arrays
+                        properties:
+                          id:
+                            type: integer
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }'
+  - match: { errors: false }
+
+  - do:
+      search:
+        index: test
+        sort: name
+  - match:  { hits.hits.0._source.name: A }
+  - length: { hits.hits.0._source.path.to.some: 2}
+  - match: { hits.hits.0._source.path.to.some.0.id: 10 }
+  - match: { hits.hits.0._source.path.to.some.1.id: [ 1, 3, 2] }
+
+
 ---
 # 112156
 stored field under object with store_array_source:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_alias/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_alias/10_basic.yml
index bf499de8463bd..a4223c2a983be 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_alias/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_alias/10_basic.yml
@@ -34,17 +34,3 @@
         name: test_alias
 
   - is_false: ''
-
----
-"Test indices.exists_alias with local flag":
-  - skip:
-      features: ["allowed_warnings"]
-
-  - do:
-      indices.exists_alias:
-        name: test_alias
-        local: true
-      allowed_warnings:
-        - "the [?local=true] query parameter to get-aliases requests has no effect and will be removed in a future version"
-
-  - is_false: ''
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml
index 4f26a69712e83..63ab40f3bf578 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml
@@ -289,21 +289,6 @@ setup:
         index: non-existent
         name: foo
 
----
-"Get alias with local flag":
-  - skip:
-      features: ["allowed_warnings"]
-
-  - do:
-      indices.get_alias:
-        local: true
-      allowed_warnings:
-        - "the [?local=true] query parameter to get-aliases requests has no effect and will be removed in a future version"
-
-  - is_true: test_index
-
-  - is_true: test_index_2
-
 ---
 "Get alias against closed indices":
   - skip:
@@ -329,17 +314,3 @@ setup:
 
   - is_true: test_index
   - is_false: test_index_2
-
-
----
-"Deprecated local parameter":
-  - requires:
-      cluster_features: "gte_v8.12.0"
-      test_runner_features: ["warnings"]
-      reason: verifying deprecation warnings from 8.12.0 onwards
-
-  - do:
-      indices.get_alias:
-        local: true
-      warnings:
-        - "the [?local=true] query parameter to get-aliases requests has no effect and will be removed in a future version"
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
index d209c839d904b..b4709a4e4d176 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
@@ -1,21 +1,29 @@
 ---
-stored _source mode is not supported:
+synthetic _source is default:
   - requires:
-      test_runner_features: [capabilities]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [logsdb_index_mode]
-      reason: "Support for 'logsdb' index mode capability required"
-
-  - skip:
-      known_issues:
-        - cluster_feature: "gte_v8.15.0"
-          fixed_by: "gte_v8.16.0"
-      reason: "Development of logs index mode spans 8.15 and 8.16"
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new validation logic
+
+  - do:
+      indices.create:
+        index: test-default-source
+        body:
+          settings:
+            index:
+              mode: logsdb
+  - do:
+      indices.get:
+        index: test-default-source
+
+  - match: { test-default-source.mappings._source.mode: "synthetic" }
+
+---
+stored _source mode is supported:
+  - requires:
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new validation logic
 
   - do:
-      catch: bad_request
       indices.create:
         index: test-stored-source
         body:
@@ -25,31 +33,17 @@ stored _source mode is not supported:
           mappings:
             _source:
               mode: stored
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
+  - do:
+      indices.get:
+        index: test-stored-source
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { test-stored-source.mappings._source.mode: "stored" }
 
 ---
 disabled _source is not supported:
   - requires:
-      test_runner_features: [capabilities]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [logsdb_index_mode]
-      reason: "Support for 'logsdb' index mode capability required"
-
-  - skip:
-      known_issues:
-        - cluster_feature: "gte_v8.15.0"
-          fixed_by: "gte_v8.16.0"
-      reason: "Development of logs index mode spans 8.15 and 8.16"
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new error message
 
   - do:
       catch: bad_request
@@ -62,20 +56,15 @@ disabled _source is not supported:
           mappings:
             _source:
               enabled: false
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
 
   - match: { error.type: "mapper_parsing_exception" }
   - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
 
   - do:
       catch: bad_request
       indices.create:
-        index: test-disabled-source
+        index: test-disabled-mode-source
         body:
           settings:
             index:
@@ -83,12 +72,81 @@ disabled _source is not supported:
           mappings:
             _source:
               mode: disabled
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
 
   - match: { error.type: "mapper_parsing_exception" }
   - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
+
+---
+include/exclude is not supported with synthetic _source:
+  - requires:
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new validation logic
+
+  - do:
+      catch: '/filtering the stored _source is incompatible with synthetic source/'
+      indices.create:
+        index: test-includes
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              includes: [a]
+
+  - do:
+      catch: '/filtering the stored _source is incompatible with synthetic source/'
+      indices.create:
+        index: test-excludes
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              excludes: [b]
+
+---
+include/exclude is supported with stored _source:
+  - requires:
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new validation logic
+
+  - do:
+      indices.create:
+        index: test-includes
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              mode: stored
+              includes: [a]
+
+  - do:
+      indices.get:
+        index: test-includes
+
+  - match: { test-includes.mappings._source.mode: "stored" }
+  - match: { test-includes.mappings._source.includes: ["a"] }
+
+  - do:
+      indices.create:
+        index: test-excludes
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              mode: stored
+              excludes: [b]
+
+  - do:
+      indices.get:
+        index: test-excludes
+
+  - match: { test-excludes.mappings._source.mode: "stored" }
+  - match: { test-excludes.mappings._source.excludes: ["b"] }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml
new file mode 100644
index 0000000000000..188c155e4a836
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml
@@ -0,0 +1,160 @@
+setup:
+  - requires:
+      cluster_features: "mapper.vectors.bbq"
+      reason: 'kNN float to better-binary quantization is required'
+  - do:
+      indices.create:
+        index: bbq_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              name:
+                type: keyword
+              vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: bbq_hnsw
+              another_vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: bbq_hnsw
+
+  - do:
+      index:
+        index: bbq_hnsw
+        id: "1"
+        body:
+          name: cow.jpg
+          vector: [300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0]
+          another_vector: [115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0]
+  # Flush in order to provoke a merge later
+  - do:
+      indices.flush:
+        index: bbq_hnsw
+
+  - do:
+      index:
+        index: bbq_hnsw
+        id: "2"
+        body:
+          name: moose.jpg
+          vector: [100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0]
+          another_vector: [50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120]
+  # Flush in order to provoke a merge later
+  - do:
+      indices.flush:
+        index: bbq_hnsw
+
+  - do:
+      index:
+        index: bbq_hnsw
+        id: "3"
+        body:
+          name: rabbit.jpg
+          vector: [111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0]
+          another_vector: [11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0]
+  # Flush in order to provoke a merge later
+  - do:
+      indices.flush:
+        index: bbq_hnsw
+
+  - do:
+      indices.forcemerge:
+        index: bbq_hnsw
+        max_num_segments: 1
+---
+"Test knn search":
+  - do:
+      search:
+        index: bbq_hnsw
+        body:
+          knn:
+            field: vector
+            query_vector: [ 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0]
+            k: 3
+            num_candidates: 3
+
+  # Depending on how things are distributed, docs 2 and 3 might be swapped
+  # here we verify that are last hit is always the worst one
+  - match: { hits.hits.2._id: "1" }
+
+---
+"Test bad quantization parameters":
+  - do:
+      catch: bad_request
+      indices.create:
+        index: bad_bbq_hnsw
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 64
+                element_type: byte
+                index: true
+                index_options:
+                  type: bbq_hnsw
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: bad_bbq_hnsw
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 64
+                index: false
+                index_options:
+                  type: bbq_hnsw
+---
+"Test few dimensions fail indexing":
+  - do:
+      catch: bad_request
+      indices.create:
+        index: bad_bbq_hnsw
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 42
+                index: true
+                index_options:
+                  type: bbq_hnsw
+
+  - do:
+      indices.create:
+        index: dynamic_dim_bbq_hnsw
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: bbq_hnsw
+
+  - do:
+      catch: bad_request
+      index:
+        index: dynamic_dim_bbq_hnsw
+        body:
+          vector: [1.0, 2.0, 3.0, 4.0, 5.0]
+
+  - do:
+      index:
+        index: dynamic_dim_bbq_hnsw
+        body:
+          vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml
new file mode 100644
index 0000000000000..ed7a8dd5df65d
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml
@@ -0,0 +1,165 @@
+setup:
+  - requires:
+      cluster_features: "mapper.vectors.bbq"
+      reason: 'kNN float to better-binary quantization is required'
+  - do:
+      indices.create:
+        index: bbq_flat
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              name:
+                type: keyword
+              vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: bbq_flat
+              another_vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: bbq_flat
+
+  - do:
+      index:
+        index: bbq_flat
+        id: "1"
+        body:
+          name: cow.jpg
+          vector: [300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0]
+          another_vector: [115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0]
+  # Flush in order to provoke a merge later
+  - do:
+      indices.flush:
+        index: bbq_flat
+
+  - do:
+      index:
+        index: bbq_flat
+        id: "2"
+        body:
+          name: moose.jpg
+          vector: [100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0]
+          another_vector: [50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120]
+  # Flush in order to provoke a merge later
+  - do:
+      indices.flush:
+        index: bbq_flat
+
+  - do:
+      index:
+        index: bbq_flat
+        id: "3"
+        body:
+          name: rabbit.jpg
+          vector: [111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0]
+          another_vector: [11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0]
+  # Flush in order to provoke a merge later
+  - do:
+      indices.flush:
+        index: bbq_flat
+
+  - do:
+      indices.forcemerge:
+        index: bbq_flat
+        max_num_segments: 1
+---
+"Test knn search":
+  - do:
+      search:
+        index: bbq_flat
+        body:
+          knn:
+            field: vector
+            query_vector: [ 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0]
+            k: 3
+            num_candidates: 3
+
+  # Depending on how things are distributed, docs 2 and 3 might be swapped
+  # here we verify that are last hit is always the worst one
+  - match: { hits.hits.2._id: "1" }
+---
+"Test bad parameters":
+  - do:
+      catch: bad_request
+      indices.create:
+        index: bad_bbq_flat
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                index_options:
+                  type: bbq_flat
+                  m: 42
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: bad_bbq_flat
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 64
+                element_type: byte
+                index: true
+                index_options:
+                  type: bbq_flat
+---
+"Test few dimensions fail indexing":
+  # verify index creation fails
+  - do:
+      catch: bad_request
+      indices.create:
+        index: bad_bbq_flat
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 42
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: bbq_flat
+
+  # verify dynamic dimension fails
+  - do:
+      indices.create:
+        index: dynamic_dim_bbq_flat
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: bbq_flat
+
+  # verify index fails for odd dim vector
+  - do:
+      catch: bad_request
+      index:
+        index: dynamic_dim_bbq_flat
+        body:
+          vector: [1.0, 2.0, 3.0, 4.0, 5.0]
+
+  # verify that we can index an even dim vector after the odd dim vector failure
+  - do:
+      index:
+        index: dynamic_dim_bbq_flat
+        body:
+          vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml
index ed469ffd7ff16..02576ad1b2b01 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml
@@ -354,3 +354,54 @@ setup:
               dims: 40
               index: true
               similarity: max_inner_product
+
+
+---
+"Search with synthetic source":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ bit_dense_vector_synthetic_source ]
+      test_runner_features: capabilities
+      reason: "Support for bit dense vector synthetic source capability required"
+  - do:
+      indices.create:
+        index: test_synthetic_source
+        body:
+          mappings:
+            properties:
+              name:
+                type: keyword
+              vector1:
+                type: dense_vector
+                element_type: bit
+                dims: 40
+                index: false
+              vector2:
+                type: dense_vector
+                element_type: bit
+                dims: 40
+                index: true
+                similarity: l2_norm
+
+  - do:
+      index:
+        index: test_synthetic_source
+        id: "1"
+        body:
+          name: cow.jpg
+          vector1: [2, -1, 1, 4, -3]
+          vector2: [2, -1, 1, 4, -3]
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        force_synthetic_source: true
+        index: test_synthetic_source
+
+  - match: {hits.hits.0._id: "1"}
+  - match: {hits.hits.0._source.vector1: [2, -1, 1, 4, -3]}
+  - match: {hits.hits.0._source.vector2: [2, -1, 1, 4, -3]}
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml
index dc79961ae78cd..81ca84a06f815 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml
@@ -212,7 +212,6 @@ dfs knn vector profiling:
 
   - match: { hits.total.value: 1 }
   - match: { profile.shards.0.dfs.knn.0.query.0.type: "DocAndScoreQuery" }
-  - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScore[100]" }
   - gt: { profile.shards.0.dfs.knn.0.query.0.time_in_nanos: 0 }
   - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.set_min_competitive_score_count: 0 }
   - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.set_min_competitive_score: 0 }
@@ -235,6 +234,47 @@ dfs knn vector profiling:
   - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" }
   - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 }
 
+---
+dfs knn vector profiling description:
+  - requires:
+      cluster_features: ["lucene_10_upgrade"]
+      reason: "the profile description changed with Lucene 10"
+  - do:
+      indices.create:
+        index: images
+        body:
+          settings:
+            index.number_of_shards: 1
+          mappings:
+            properties:
+              image:
+                type: "dense_vector"
+                dims: 3
+                index: true
+                similarity: "l2_norm"
+
+  - do:
+      index:
+        index: images
+        id: "1"
+        refresh: true
+        body:
+          image: [1, 5, -20]
+
+  - do:
+      search:
+        index: images
+        body:
+          profile: true
+          knn:
+            field: "image"
+            query_vector: [-5, 9, -12]
+            k: 1
+            num_candidates: 100
+
+  - match: { hits.total.value: 1 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScoreQuery[0,...][0.009673266,...],0.009673266" }
+
 ---
 dfs knn vector profiling with vector_operations_count:
   - requires:
@@ -276,7 +316,6 @@ dfs knn vector profiling with vector_operations_count:
 
   - match: { hits.total.value: 1 }
   - match: { profile.shards.0.dfs.knn.0.query.0.type: "DocAndScoreQuery" }
-  - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScore[100]" }
   - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 1 }
   - gt: { profile.shards.0.dfs.knn.0.query.0.time_in_nanos: 0 }
   - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.set_min_competitive_score_count: 0 }
@@ -300,7 +339,6 @@ dfs knn vector profiling with vector_operations_count:
   - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" }
   - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 }
 
-
 ---
 dfs profile for search with dfs_query_then_fetch:
   - requires:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml
index bcd58f3f7bd64..675b98133ce11 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml
@@ -15,6 +15,10 @@ setup:
 
   - match: { result: "created" }
 
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       synonyms.get_synonym:
         id: test-update-synonyms
@@ -58,6 +62,10 @@ setup:
 
   - match: { result: "created" }
 
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       synonyms.get_synonym:
         id: test-empty-synonyms
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml
index d3d0a3bb4df70..4e77e10495109 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml
@@ -11,6 +11,11 @@ setup:
           synonyms_set:
             synonyms: "foo => bar, baz"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       indices.create:
         index: test_index
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml
index 3494f33466ce4..5e6d4ec2341ad 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml
@@ -14,6 +14,10 @@ setup:
             - synonyms: "test => check"
               id: "test-id-3"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
 
 ---
 "Get synonyms set":
@@ -31,7 +35,6 @@ setup:
           id: "test-id-2"
         - synonyms: "test => check"
           id: "test-id-3"
-
 ---
 "Get synonyms set - not found":
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml
index 351ff4e186d8a..23c907f6a1137 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml
@@ -12,6 +12,10 @@ setup:
             - synonyms: "bye => goodbye"
               id: "test-id-2"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
 ---
 "Delete synonyms set":
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml
index 723c41e163eb8..7c145dafd81cd 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml
@@ -9,6 +9,12 @@ setup:
           synonyms_set:
             - synonyms: "hello, hi"
             - synonyms: "goodbye, bye"
+
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   - do:
       synonyms.put_synonym:
         id: test-synonyms-1
@@ -23,21 +29,8 @@ setup:
         body:
           synonyms_set:
             - synonyms: "pc, computer"
-  # set logging to debug for issue: https://github.com/elastic/elasticsearch/issues/102261
-  - do:
-      cluster.put_settings:
-        body:
-          persistent:
-            logger.org.elasticsearch.synonyms: DEBUG
 
 ---
-teardown:
-  - do:
-      cluster.put_settings:
-        body:
-          persistent:
-            logger.org.elasticsearch.synonyms: null
----
 "List synonyms set":
   - do:
       synonyms.get_synonyms_sets: { }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml
index f3711bb0774ca..d8611000fe465 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml
@@ -14,7 +14,10 @@ setup:
             - synonyms: "test => check"
               id: "test-id-3"
 
-
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
 ---
 "Update a synonyms rule":
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml
index 2a7c8aff89d8e..0c962b51e08cb 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml
@@ -13,11 +13,12 @@ setup:
               id: "test-id-2"
             - synonyms: "test => check"
               id: "test-id-3"
+
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
   - do:
       cluster.health:
-        index: .synonyms
-        timeout: 1m
-        wait_for_status: green
+        wait_for_no_initializing_shards: true
+
 
 ---
 "Get a synonym rule":
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml
index a4853b0b6d414..41ab293158a35 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml
@@ -14,6 +14,11 @@ setup:
             - synonyms: "test => check"
               id: "test-id-3"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
 ---
 "Delete synonym rule":
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml
index 89ad933370e1c..3aba0f0b4b78b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml
@@ -2,7 +2,6 @@ setup:
   - requires:
       cluster_features: ["gte_v8.10.0"]
       reason: Loading synonyms from index is introduced in 8.10.0
-
   # Create a new synonyms set
   - do:
       synonyms.put_synonym:
@@ -14,6 +13,11 @@ setup:
             - synonyms: "bye => goodbye"
               id: "synonym-rule-2"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   # Create an index with synonym_filter that uses that synonyms set
   - do:
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml
index dc94b36222402..1ceb5b43b8129 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml
@@ -3,7 +3,6 @@
   - requires:
       cluster_features: ["gte_v8.10.0"]
       reason: Reloading analyzers for specific synonym set is introduced in 8.10.0
-
   # Create synonyms_set1
   - do:
       synonyms.put_synonym:
@@ -26,6 +25,11 @@
             - synonyms: "bye => goodbye"
               id: "synonym-rule-2"
 
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
   # Create my_index1 with synonym_filter that uses synonyms_set1
   - do:
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
index ade153d284548..c5669cd6414b1 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
@@ -456,13 +456,12 @@ nested fields:
   - match: {tsdb-synthetic.mappings._source.mode: synthetic}
 
 ---
-regular source:
+stored source is supported:
   - requires:
-      cluster_features: ["gte_v8.7.0"]
-      reason: synthetic source
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new validation logic
 
   - do:
-      catch: '/time series indices only support synthetic source/'
       indices.create:
         index: tsdb_index
         body:
@@ -486,14 +485,21 @@ regular source:
                       uid:
                         type: keyword
                         time_series_dimension: true
+
+  - do:
+      indices.get:
+        index: tsdb_index
+
+  - match: { tsdb_index.mappings._source.mode: "stored" }
+
 ---
-disabled source:
+disabled source is not supported:
   - requires:
-      cluster_features: ["gte_v8.7.0"]
-      reason: synthetic source
+      cluster_features: ["mapper.source.remove_synthetic_source_only_validation"]
+      reason: requires new error message
 
   - do:
-      catch: '/time series indices only support synthetic source/'
+      catch: bad_request
       indices.create:
         index: tsdb_index
         body:
@@ -518,6 +524,40 @@ disabled source:
                         type: keyword
                         time_series_dimension: true
 
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              enabled: false
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
+
 ---
 source include/exclude:
   - requires:
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java
index c56bc201e7f86..8bedf436e3698 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java
@@ -571,7 +571,7 @@ public void testSearchQueryThenFetch() throws Exception {
         SearchRequest searchRequest = new SearchRequest(randomIndicesOrAliases).searchType(SearchType.QUERY_THEN_FETCH);
         assertNoFailuresAndResponse(
             internalCluster().coordOnlyNodeClient().search(searchRequest),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L))
         );
 
         clearInterceptedActions();
@@ -601,7 +601,7 @@ public void testSearchDfsQueryThenFetch() throws Exception {
         SearchRequest searchRequest = new SearchRequest(randomIndicesOrAliases).searchType(SearchType.DFS_QUERY_THEN_FETCH);
         assertNoFailuresAndResponse(
             internalCluster().coordOnlyNodeClient().search(searchRequest),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L))
         );
 
         clearInterceptedActions();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index cc6329a973b37..e8160a311bedb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -306,8 +306,8 @@ public void onFailure(Exception e) {
             prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setQuery(new RangeQueryBuilder("index_version").from(indexVersion.get(), true)),
             expected -> assertNoFailuresAndResponse(prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()), all -> {
-                assertEquals(expected + " vs. " + all, expected.getHits().getTotalHits().value, all.getHits().getTotalHits().value);
-                logger.info("total: {}", expected.getHits().getTotalHits().value);
+                assertEquals(expected + " vs. " + all, expected.getHits().getTotalHits().value(), all.getHits().getTotalHits().value());
+                logger.info("total: {}", expected.getHits().getTotalHits().value());
             })
         );
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
index e1bf5bce6f3ae..8391ab270b1d1 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
@@ -253,7 +253,7 @@ public void assertNested(String index, int numDocs) {
         // now, do a nested query
         assertNoFailuresAndResponse(
             prepareSearch(index).setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"), ScoreMode.Avg)),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) numDocs))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numDocs))
         );
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java
index 8b8b62da98f97..2fd6ee9a16808 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java
@@ -141,11 +141,11 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure)
         assertResponse(prepareSearch(INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).setSize(0), results -> {
             assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L));
             if (rejectedExecutionExpected) {
-                assertThat((int) results.getHits().getTotalHits().value, lessThanOrEqualTo(numberOfAsyncOps));
+                assertThat((int) results.getHits().getTotalHits().value(), lessThanOrEqualTo(numberOfAsyncOps));
             } else if (finalRejectedAfterAllRetries) {
-                assertThat((int) results.getHits().getTotalHits().value, lessThan(numberOfAsyncOps));
+                assertThat((int) results.getHits().getTotalHits().value(), lessThan(numberOfAsyncOps));
             } else {
-                assertThat((int) results.getHits().getTotalHits().value, equalTo(numberOfAsyncOps));
+                assertThat((int) results.getHits().getTotalHits().value(), equalTo(numberOfAsyncOps));
             }
         });
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
index 37904e9f639ac..4ed19065f32f2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
@@ -136,11 +136,11 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
         final boolean finalRejectedAfterAllRetries = rejectedAfterAllRetries;
         assertResponse(prepareSearch(INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).setSize(0), results -> {
             if (rejectedExecutionExpected) {
-                assertThat((int) results.getHits().getTotalHits().value, lessThanOrEqualTo(numberOfAsyncOps));
+                assertThat((int) results.getHits().getTotalHits().value(), lessThanOrEqualTo(numberOfAsyncOps));
             } else if (finalRejectedAfterAllRetries) {
-                assertThat((int) results.getHits().getTotalHits().value, lessThan(numberOfAsyncOps));
+                assertThat((int) results.getHits().getTotalHits().value(), lessThan(numberOfAsyncOps));
             } else {
-                assertThat((int) results.getHits().getTotalHits().value, equalTo(numberOfAsyncOps));
+                assertThat((int) results.getHits().getTotalHits().value(), equalTo(numberOfAsyncOps));
             }
         });
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java
index cde8d41b292b7..4977d87d5a348 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java
@@ -90,7 +90,7 @@ public void testSingleBulkRequest() {
 
         assertResponse(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), searchResponse -> {
             assertNoFailures(searchResponse);
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) 1));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) 1));
         });
 
         assertFalse(refCounted.hasReferences());
@@ -268,7 +268,7 @@ public void testMultipleBulkPartsWithBackoff() {
 
             assertResponse(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), searchResponse -> {
                 assertNoFailures(searchResponse);
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(docs));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(docs));
             });
         }
     }
@@ -358,7 +358,7 @@ public void testBulkLevelBulkFailureAfterFirstIncrementalRequest() throws Except
 
             assertResponse(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), searchResponse -> {
                 assertNoFailures(searchResponse);
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(hits.get()));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(hits.get()));
             });
         }
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java
index af99a0344e030..d5d21c548a15d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java
@@ -34,6 +34,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -59,7 +60,7 @@ public void testMappingValidationIndexExists() {
             }
             """;
         indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet();
-        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
         bulkRequest.add(new IndexRequest(indexName).source("""
             {
               "foo1": "baz"
@@ -81,7 +82,7 @@ public void testMappingValidationIndexExists() {
         );
         indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet();
         SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet();
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
+        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(0L));
         searchResponse.decRef();
         ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         Map<String, Object> indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap();
@@ -131,14 +132,14 @@ public void testMappingValidationIndexExistsTemplateSubstitutions() throws IOExc
 
         String indexName = "my-index-1";
         // First, run before the index is created:
-        assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName);
+        assertMappingsUpdatedFromSubstitutions(indexName, indexTemplateName);
         // Now, create the index and make sure the component template substitutions work the same:
         indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet();
-        assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName);
+        assertMappingsUpdatedFromSubstitutions(indexName, indexTemplateName);
         // Now make sure nothing was actually changed:
         indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet();
         SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet();
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
+        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(0L));
         searchResponse.decRef();
         ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         Map<String, Object> indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap();
@@ -146,7 +147,7 @@ public void testMappingValidationIndexExistsTemplateSubstitutions() throws IOExc
         assertThat(fields.size(), equalTo(1));
     }
 
-    private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName, String indexTemplateName) {
+    private void assertMappingsUpdatedFromSubstitutions(String indexName, String indexTemplateName) {
         IndexRequest indexRequest1 = new IndexRequest(indexName).source("""
             {
               "foo1": "baz"
@@ -159,7 +160,7 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde
             """, XContentType.JSON).id(randomUUID());
         {
             // First we use the original component template, and expect a failure in the second document:
-            BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+            BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
             bulkRequest.add(indexRequest1);
             bulkRequest.add(indexRequest2);
             BulkResponse response = client().execute(new ActionType<BulkResponse>(SimulateBulkAction.NAME), bulkRequest).actionGet();
@@ -192,6 +193,7 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde
                         )
                     )
                 ),
+                Map.of(),
                 Map.of()
             );
             bulkRequest.add(indexRequest1);
@@ -226,7 +228,34 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde
                         )
                     )
                 ),
-                Map.of(indexTemplateName, Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2")))
+                Map.of(
+                    indexTemplateName,
+                    Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2"))
+                ),
+                Map.of()
+            );
+            bulkRequest.add(indexRequest1);
+            bulkRequest.add(indexRequest2);
+            BulkResponse response = client().execute(new ActionType<BulkResponse>(SimulateBulkAction.NAME), bulkRequest).actionGet();
+            assertThat(response.getItems().length, equalTo(2));
+            assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED));
+            assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException());
+            assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED));
+            assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException());
+        }
+
+        {
+            /*
+             * Now we mapping_addition that defines both fields, so we expect no exception:
+             */
+            BulkRequest bulkRequest = new SimulateBulkRequest(
+                Map.of(),
+                Map.of(),
+                Map.of(),
+                Map.of(
+                    "_doc",
+                    Map.of("dynamic", "strict", "properties", Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")))
+                )
             );
             bulkRequest.add(indexRequest1);
             bulkRequest.add(indexRequest2);
@@ -245,7 +274,7 @@ public void testMappingValidationIndexDoesNotExistsNoTemplate() {
          * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure.
          */
         String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
-        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
         bulkRequest.add(new IndexRequest(indexName).source("""
             {
               "foo1": "baz"
@@ -292,7 +321,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti
         request.indexTemplate(composableIndexTemplate);
 
         client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
-        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
         bulkRequest.add(new IndexRequest(indexName).source("""
             {
               "foo1": "baz"
@@ -324,7 +353,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() {
         indicesAdmin().putTemplate(
             new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer")
         ).actionGet();
-        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
         bulkRequest.add(new IndexRequest(indexName).source("""
             {
               "foo1": "baz"
@@ -378,7 +407,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti
         client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
         {
             // First, try with no @timestamp to make sure we're picking up data-stream-specific templates
-            BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+            BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
             bulkRequest.add(new IndexRequest(indexName).source("""
                 {
                   "foo1": "baz"
@@ -389,7 +418,8 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti
                   "foo3": "baz"
                 }
                 """, XContentType.JSON).id(randomUUID()));
-            BulkResponse response = client().execute(new ActionType<BulkResponse>(SimulateBulkAction.NAME), bulkRequest).actionGet();
+            BulkResponse response = client().execute(new ActionType<BulkResponse>(SimulateBulkAction.NAME), bulkRequest)
+                .actionGet(5, TimeUnit.SECONDS);
             assertThat(response.getItems().length, equalTo(2));
             assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED));
             assertThat(
@@ -404,7 +434,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti
         }
         {
             // Now with @timestamp
-            BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+            BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
             bulkRequest.add(new IndexRequest(indexName).source("""
                 {
                   "@timestamp": "2024-08-27",
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java
index 274cf90ec9529..f17196c3d97f1 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java
@@ -45,9 +45,9 @@ public void testIndexWithWriteDelayEnabled() throws Exception {
             try {
                 logger.debug("running search");
                 assertResponse(prepareSearch("test"), response -> {
-                    if (response.getHits().getTotalHits().value != numOfDocs) {
+                    if (response.getHits().getTotalHits().value() != numOfDocs) {
                         final String message = "Count is "
-                            + response.getHits().getTotalHits().value
+                            + response.getHits().getTotalHits().value()
                             + " but "
                             + numOfDocs
                             + " was expected. "
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java
index 66323e687eefb..e47925cef913b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java
@@ -612,7 +612,7 @@ public void testMissingShardsWithPointInTime() throws Exception {
                 assertThat(resp.getSuccessfulShards(), equalTo(numShards - shardsRemoved));
                 assertThat(resp.getFailedShards(), equalTo(shardsRemoved));
                 assertNotNull(resp.getHits().getTotalHits());
-                assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs));
+                assertThat(resp.getHits().getTotalHits().value(), lessThan((long) numDocs));
             });
 
             // create a PIT when some shards are missing
@@ -637,7 +637,7 @@ public void testMissingShardsWithPointInTime() throws Exception {
                         assertThat(resp.getFailedShards(), equalTo(shardsRemoved));
                         assertThat(resp.pointInTimeId(), equalTo(pointInTimeResponseOneNodeDown.getPointInTimeId()));
                         assertNotNull(resp.getHits().getTotalHits());
-                        assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs));
+                        assertThat(resp.getHits().getTotalHits().value(), lessThan((long) numDocs));
                     }
                 );
 
@@ -661,7 +661,7 @@ public void testMissingShardsWithPointInTime() throws Exception {
                         assertThat(resp.getSuccessfulShards(), equalTo(numShards));
                         assertThat(resp.getFailedShards(), equalTo(0));
                         assertNotNull(resp.getHits().getTotalHits());
-                        assertThat(resp.getHits().getTotalHits().value, greaterThan((long) numDocs));
+                        assertThat(resp.getHits().getTotalHits().value(), greaterThan((long) numDocs));
                     });
 
                     // ensure that when using the previously created PIT, we'd see the same number of documents as before regardless of the
@@ -681,7 +681,7 @@ public void testMissingShardsWithPointInTime() throws Exception {
                             }
                             assertNotNull(resp.getHits().getTotalHits());
                             // we expect less documents as the newly indexed ones should not be part of the PIT
-                            assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs));
+                            assertThat(resp.getHits().getTotalHits().value(), lessThan((long) numDocs));
                         }
                     );
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
index d1a68c68e7de5..a1395f81eb091 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
@@ -143,7 +143,7 @@ public void testLocalClusterAlias() throws ExecutionException, InterruptedExcept
                 randomBoolean()
             );
             assertResponse(client().search(searchRequest), searchResponse -> {
-                assertEquals(1, searchResponse.getHits().getTotalHits().value);
+                assertEquals(1, searchResponse.getHits().getTotalHits().value());
                 SearchHit[] hits = searchResponse.getHits().getHits();
                 assertEquals(1, hits.length);
                 SearchHit hit = hits[0];
@@ -162,7 +162,7 @@ public void testLocalClusterAlias() throws ExecutionException, InterruptedExcept
                 randomBoolean()
             );
             assertResponse(client().search(searchRequest), searchResponse -> {
-                assertEquals(1, searchResponse.getHits().getTotalHits().value);
+                assertEquals(1, searchResponse.getHits().getTotalHits().value());
                 SearchHit[] hits = searchResponse.getHits().getHits();
                 assertEquals(1, hits.length);
                 SearchHit hit = hits[0];
@@ -221,7 +221,7 @@ public void testAbsoluteStartMillis() throws ExecutionException, InterruptedExce
             );
             searchRequest.indices("<test-{now/d}>");
             assertResponse(client().search(searchRequest), searchResponse -> {
-                assertEquals(1, searchResponse.getHits().getTotalHits().value);
+                assertEquals(1, searchResponse.getHits().getTotalHits().value());
                 assertEquals("test-1970.01.01", searchResponse.getHits().getHits()[0].getIndex());
             });
         }
@@ -241,7 +241,7 @@ public void testAbsoluteStartMillis() throws ExecutionException, InterruptedExce
             sourceBuilder.query(rangeQuery);
             searchRequest.source(sourceBuilder);
             assertResponse(client().search(searchRequest), searchResponse -> {
-                assertEquals(1, searchResponse.getHits().getTotalHits().value);
+                assertEquals(1, searchResponse.getHits().getTotalHits().value());
                 assertEquals("test-1970.01.01", searchResponse.getHits().getHits()[0].getIndex());
             });
         }
@@ -280,7 +280,7 @@ public void testFinalReduce() throws ExecutionException, InterruptedException {
                 ? originalRequest
                 : SearchRequest.subSearchRequest(taskId, originalRequest, Strings.EMPTY_ARRAY, "remote", nowInMillis, true);
             assertResponse(client().search(searchRequest), searchResponse -> {
-                assertEquals(2, searchResponse.getHits().getTotalHits().value);
+                assertEquals(2, searchResponse.getHits().getTotalHits().value());
                 InternalAggregations aggregations = searchResponse.getAggregations();
                 LongTerms longTerms = aggregations.get("terms");
                 assertEquals(1, longTerms.getBuckets().size());
@@ -296,7 +296,7 @@ public void testFinalReduce() throws ExecutionException, InterruptedException {
                 false
             );
             assertResponse(client().search(searchRequest), searchResponse -> {
-                assertEquals(2, searchResponse.getHits().getTotalHits().value);
+                assertEquals(2, searchResponse.getHits().getTotalHits().value());
                 InternalAggregations aggregations = searchResponse.getAggregations();
                 LongTerms longTerms = aggregations.get("terms");
                 assertEquals(2, longTerms.getBuckets().size());
@@ -432,7 +432,7 @@ public void testSearchIdle() throws Exception {
             () -> assertResponse(
                 prepareSearch("test").setQuery(new RangeQueryBuilder("created_date").gte("2020-01-02").lte("2020-01-03"))
                     .setPreFilterShardSize(randomIntBetween(1, 3)),
-                resp -> assertThat(resp.getHits().getTotalHits().value, equalTo(2L))
+                resp -> assertThat(resp.getHits().getTotalHits().value(), equalTo(2L))
             )
         );
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java
index 848c5cacda1b9..b70da34c8fe3f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -396,7 +396,7 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("foos").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(2L))
         );
 
         logger.info("--> checking filtering alias for one index");
@@ -406,7 +406,7 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("bars").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L))
         );
 
         logger.info("--> checking filtering alias for two indices and one complete index");
@@ -416,7 +416,7 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("foos", "test1").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(5L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(5L))
         );
 
         logger.info("--> checking filtering alias for two indices and non-filtering alias for one index");
@@ -426,17 +426,17 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("foos", "aliasToTest1").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(5L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(5L))
         );
 
         logger.info("--> checking filtering alias for two indices and non-filtering alias for both indices");
         assertResponse(
             prepareSearch("foos", "aliasToTests").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(8L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(8L))
         );
         assertResponse(
             prepareSearch("foos", "aliasToTests").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(8L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(8L))
         );
 
         logger.info("--> checking filtering alias for two indices and non-filtering alias for both indices");
@@ -446,7 +446,7 @@ public void testSearchingFilteringAliasesTwoIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("foos", "aliasToTests").setSize(0).setQuery(QueryBuilders.termQuery("name", "something")),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(2L))
         );
     }
 
@@ -508,7 +508,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("filter23", "filter13").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(4L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(4L))
         );
 
         assertResponse(
@@ -517,7 +517,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("filter23", "filter1").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(5L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(5L))
         );
 
         assertResponse(
@@ -526,7 +526,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("filter13", "filter1").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(4L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(4L))
         );
 
         assertResponse(
@@ -535,7 +535,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("filter13", "filter1", "filter23").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(6L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(6L))
         );
 
         assertResponse(
@@ -544,7 +544,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("filter23", "filter13", "test2").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(6L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(6L))
         );
 
         assertResponse(
@@ -553,7 +553,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
         );
         assertResponse(
             prepareSearch("filter23", "filter13", "test1", "test2").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(8L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(8L))
         );
     }
 
@@ -608,7 +608,7 @@ public void testDeletingByQueryFilteringAliases() throws Exception {
         logger.info("--> checking counts before delete");
         assertResponse(
             prepareSearch("bars").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L))
         );
     }
 
@@ -1399,7 +1399,7 @@ private void checkAliases() {
     }
 
     private void assertHits(SearchHits hits, String... ids) {
-        assertThat(hits.getTotalHits().value, equalTo((long) ids.length));
+        assertThat(hits.getTotalHits().value(), equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
         for (SearchHit hit : hits.getHits()) {
             hitIds.add(hit.getId());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java
index 4e7c22f0d8847..f7dae8a92c2d6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java
@@ -44,7 +44,7 @@ public void testBroadcastOperations() throws IOException {
         for (int i = 0; i < 5; i++) {
             // test successful
             assertResponse(prepareSearch("test").setSize(0).setQuery(matchAllQuery()), countResponse -> {
-                assertThat(countResponse.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(countResponse.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(countResponse.getTotalShards(), equalTo(numShards.numPrimaries));
                 assertThat(countResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
                 assertThat(countResponse.getFailedShards(), equalTo(0));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java
new file mode 100644
index 0000000000000..cb279c93b402e
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.routing.allocation.allocator;
+
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matcher;
+
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.not;
+
+public class DesiredBalanceReconcilerMetricsIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), TestTelemetryPlugin.class);
+    }
+
+    public void testDesiredBalanceGaugeMetricsAreOnlyPublishedByCurrentMaster() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("test").setSettings(indexSettings(2, 1)).get();
+        ensureGreen();
+
+        assertOnlyMasterIsPublishingMetrics();
+
+        // fail over and check again
+        int numFailOvers = randomIntBetween(1, 3);
+        for (int i = 0; i < numFailOvers; i++) {
+            internalCluster().restartNode(internalCluster().getMasterName());
+            ensureGreen();
+
+            assertOnlyMasterIsPublishingMetrics();
+        }
+    }
+
+    private static void assertOnlyMasterIsPublishingMetrics() {
+        String masterNodeName = internalCluster().getMasterName();
+        String[] nodeNames = internalCluster().getNodeNames();
+        for (String nodeName : nodeNames) {
+            assertMetricsAreBeingPublished(nodeName, nodeName.equals(masterNodeName));
+        }
+    }
+
+    private static void assertMetricsAreBeingPublished(String nodeName, boolean shouldBePublishing) {
+        final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, nodeName)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+        testTelemetryPlugin.resetMeter();
+        testTelemetryPlugin.collect();
+        Matcher<Collection<?>> matcher = shouldBePublishing ? not(empty()) : empty();
+        assertThat(testTelemetryPlugin.getLongGaugeMeasurement(DesiredBalanceMetrics.UNASSIGNED_SHARDS_METRIC_NAME), matcher);
+        assertThat(testTelemetryPlugin.getLongGaugeMeasurement(DesiredBalanceMetrics.TOTAL_SHARDS_METRIC_NAME), matcher);
+        assertThat(testTelemetryPlugin.getLongGaugeMeasurement(DesiredBalanceMetrics.UNDESIRED_ALLOCATION_COUNT_METRIC_NAME), matcher);
+        assertThat(testTelemetryPlugin.getDoubleGaugeMeasurement(DesiredBalanceMetrics.UNDESIRED_ALLOCATION_RATIO_METRIC_NAME), matcher);
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java
index f2441e43de8d8..ffe55387d34c6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java
@@ -88,7 +88,7 @@ public Collection<RestHandler> getRestHandlers(
             return List.of(new RestHandler() {
                 @Override
                 public List<Route> routes() {
-                    return List.of(Route.builder(RestRequest.Method.POST, "_slow").build());
+                    return List.of(new Route(RestRequest.Method.POST, "_slow"));
                 }
 
                 @Override
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java
index 214fc47222f3a..bf81200509691 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java
@@ -21,21 +21,15 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.disruption.BlockMasterServiceOnMaster;
-import org.elasticsearch.test.disruption.IntermittentLongGCDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
-import org.elasticsearch.test.disruption.SingleNodeDisruption;
 import org.elasticsearch.xcontent.XContentType;
 
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
 
 /**
  * Tests relating to the loss of the master.
@@ -43,44 +37,6 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class MasterDisruptionIT extends AbstractDisruptionTestCase {
 
-    /**
-     * Test that cluster recovers from a long GC on master that causes other nodes to elect a new one
-     */
-    public void testMasterNodeGCs() throws Exception {
-        List<String> nodes = startCluster(3);
-        // NOTE: this assume must happen after starting the cluster, so that cleanup will have something to cleanup.
-        assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20);
-
-        String oldMasterNode = internalCluster().getMasterName();
-        // a very long GC, but it's OK as we remove the disruption when it has had an effect
-        SingleNodeDisruption masterNodeDisruption = new IntermittentLongGCDisruption(random(), oldMasterNode, 100, 200, 30000, 60000);
-        internalCluster().setDisruptionScheme(masterNodeDisruption);
-        masterNodeDisruption.startDisrupting();
-
-        Set<String> oldNonMasterNodesSet = new HashSet<>(nodes);
-        oldNonMasterNodesSet.remove(oldMasterNode);
-
-        List<String> oldNonMasterNodes = new ArrayList<>(oldNonMasterNodesSet);
-
-        logger.info("waiting for nodes to de-elect master [{}]", oldMasterNode);
-        for (String node : oldNonMasterNodesSet) {
-            assertDifferentMaster(node, oldMasterNode);
-        }
-
-        logger.info("waiting for nodes to elect a new master");
-        ensureStableCluster(2, oldNonMasterNodes.get(0));
-
-        // restore GC
-        masterNodeDisruption.stopDisrupting();
-        final TimeValue waitTime = new TimeValue(DISRUPTION_HEALING_OVERHEAD.millis() + masterNodeDisruption.expectedTimeToHeal().millis());
-        ensureStableCluster(3, waitTime, false, oldNonMasterNodes.get(0));
-
-        // make sure all nodes agree on master
-        String newMaster = internalCluster().getMasterName();
-        assertThat(newMaster, not(equalTo(oldMasterNode)));
-        assertMaster(newMaster, nodes);
-    }
-
     /**
      * This test isolates the master from rest of the cluster, waits for a new master to be elected, restores the partition
      * and verifies that all node agree on the new cluster state
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
index 32c602791cca4..48db23635220c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
@@ -14,33 +14,26 @@
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService;
 import org.elasticsearch.cluster.coordination.Coordinator;
 import org.elasticsearch.cluster.coordination.FollowersChecker;
 import org.elasticsearch.cluster.coordination.LeaderChecker;
 import org.elasticsearch.cluster.coordination.MasterHistoryService;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.health.GetHealthAction;
 import org.elasticsearch.health.HealthStatus;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.disruption.LongGCDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption.NetworkLinkDisruptionType;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
-import org.elasticsearch.test.disruption.SingleNodeDisruption;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ToXContent;
@@ -50,17 +43,12 @@
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.singleton;
@@ -227,266 +215,6 @@ private void testFollowerCheckerAfterMasterReelection(NetworkLinkDisruptionType
         ensureStableCluster(3);
     }
 
-    /**
-     * Tests that emulates a frozen elected master node that unfreezes and pushes its cluster state to other nodes that already are
-     * following another elected master node. These nodes should reject this cluster state and prevent them from following the stale master.
-     */
-    public void testStaleMasterNotHijackingMajority() throws Exception {
-        assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20);
-        final List<String> nodes = internalCluster().startNodes(
-            3,
-            Settings.builder()
-                .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s")
-                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s")
-                .build()
-        );
-        ensureStableCluster(3);
-
-        // Save the current master node as old master node, because that node will get frozen
-        final String oldMasterNode = internalCluster().getMasterName();
-
-        // Simulating a painful gc by suspending all threads for a long time on the current elected master node.
-        SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), oldMasterNode);
-
-        // Save the majority side
-        final List<String> majoritySide = new ArrayList<>(nodes);
-        majoritySide.remove(oldMasterNode);
-
-        // Keeps track of the previous and current master when a master node transition took place on each node on the majority side:
-        final Map<String, List<Tuple<String, String>>> masters = Collections.synchronizedMap(new HashMap<>());
-        for (final String node : majoritySide) {
-            masters.put(node, new ArrayList<>());
-            internalCluster().getInstance(ClusterService.class, node).addListener(event -> {
-                DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode();
-                DiscoveryNode currentMaster = event.state().nodes().getMasterNode();
-                if (Objects.equals(previousMaster, currentMaster) == false) {
-                    logger.info(
-                        "--> node {} received new cluster state: {} \n and had previous cluster state: {}",
-                        node,
-                        event.state(),
-                        event.previousState()
-                    );
-                    String previousMasterNodeName = previousMaster != null ? previousMaster.getName() : null;
-                    String currentMasterNodeName = currentMaster != null ? currentMaster.getName() : null;
-                    masters.get(node).add(new Tuple<>(previousMasterNodeName, currentMasterNodeName));
-                }
-            });
-        }
-
-        final CountDownLatch oldMasterNodeSteppedDown = new CountDownLatch(1);
-        internalCluster().getInstance(ClusterService.class, oldMasterNode).addListener(event -> {
-            if (event.state().nodes().getMasterNodeId() == null) {
-                oldMasterNodeSteppedDown.countDown();
-            }
-        });
-
-        internalCluster().setDisruptionScheme(masterNodeDisruption);
-        logger.info("--> freezing node [{}]", oldMasterNode);
-        masterNodeDisruption.startDisrupting();
-
-        // Wait for majority side to elect a new master
-        assertBusy(() -> {
-            for (final Map.Entry<String, List<Tuple<String, String>>> entry : masters.entrySet()) {
-                final List<Tuple<String, String>> transitions = entry.getValue();
-                assertTrue(entry.getKey() + ": " + transitions, transitions.stream().anyMatch(transition -> transition.v2() != null));
-            }
-        });
-
-        // The old master node is frozen, but here we submit a cluster state update task that doesn't get executed, but will be queued and
-        // once the old master node un-freezes it gets executed. The old master node will send this update + the cluster state where it is
-        // flagged as master to the other nodes that follow the new master. These nodes should ignore this update.
-        internalCluster().getInstance(ClusterService.class, oldMasterNode)
-            .submitUnbatchedStateUpdateTask("sneaky-update", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return ClusterState.builder(currentState).build();
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    logger.warn("failure [sneaky-update]", e);
-                }
-            });
-
-        // Save the new elected master node
-        final String newMasterNode = internalCluster().getMasterName(majoritySide.get(0));
-        logger.info("--> new detected master node [{}]", newMasterNode);
-
-        // Stop disruption
-        logger.info("--> unfreezing node [{}]", oldMasterNode);
-        masterNodeDisruption.stopDisrupting();
-
-        oldMasterNodeSteppedDown.await(30, TimeUnit.SECONDS);
-        logger.info("--> [{}] stepped down as master", oldMasterNode);
-        ensureStableCluster(3);
-
-        assertThat(masters.size(), equalTo(2));
-        for (Map.Entry<String, List<Tuple<String, String>>> entry : masters.entrySet()) {
-            String nodeName = entry.getKey();
-            List<Tuple<String, String>> transitions = entry.getValue();
-            assertTrue(
-                "[" + nodeName + "] should not apply state from old master [" + oldMasterNode + "] but it did: " + transitions,
-                transitions.stream().noneMatch(t -> oldMasterNode.equals(t.v2()))
-            );
-        }
-        assertGreenMasterStability(internalCluster().client());
-    }
-
-    /**
-     * This helper method creates a 3-node cluster where all nodes are master-eligible, and then simulates a long GC on the master node 5
-     * times (forcing another node to be elected master 5 times). It then asserts that the master stability health indicator status is
-     * YELLOW, and that expectedMasterStabilitySymptomSubstring is contained in the symptom.
-     * @param expectedMasterStabilitySymptomSubstring A string to expect in the master stability health indicator symptom
-     * @throws Exception
-     */
-    public void testRepeatedMasterChanges(String expectedMasterStabilitySymptomSubstring) throws Exception {
-        assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20);
-        final List<String> nodes = internalCluster().startNodes(
-            3,
-            Settings.builder()
-                .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s")
-                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s")
-                .put(CoordinationDiagnosticsService.IDENTITY_CHANGES_THRESHOLD_SETTING.getKey(), 1)
-                .put(CoordinationDiagnosticsService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), 100)
-                .build()
-        );
-        ensureStableCluster(3);
-        String firstMaster = internalCluster().getMasterName();
-        // Force the master to change 2 times:
-        for (int i = 0; i < 2; i++) {
-            // Save the current master node as old master node, because that node will get frozen
-            final String oldMasterNode = internalCluster().getMasterName();
-
-            // Simulating a painful gc by suspending all threads for a long time on the current elected master node.
-            SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), oldMasterNode);
-
-            // Save the majority side
-            final List<String> majoritySide = new ArrayList<>(nodes);
-            majoritySide.remove(oldMasterNode);
-
-            // Keeps track of the previous and current master when a master node transition took place on each node on the majority side:
-            final Map<String, List<Tuple<String, String>>> masters = Collections.synchronizedMap(new HashMap<>());
-            for (final String node : majoritySide) {
-                masters.put(node, new ArrayList<>());
-                internalCluster().getInstance(ClusterService.class, node).addListener(event -> {
-                    DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode();
-                    DiscoveryNode currentMaster = event.state().nodes().getMasterNode();
-                    if (Objects.equals(previousMaster, currentMaster) == false) {
-                        logger.info(
-                            "--> node {} received new cluster state: {} \n and had previous cluster state: {}",
-                            node,
-                            event.state(),
-                            event.previousState()
-                        );
-                        String previousMasterNodeName = previousMaster != null ? previousMaster.getName() : null;
-                        String currentMasterNodeName = currentMaster != null ? currentMaster.getName() : null;
-                        masters.get(node).add(new Tuple<>(previousMasterNodeName, currentMasterNodeName));
-                    }
-                });
-            }
-
-            final CountDownLatch oldMasterNodeSteppedDown = new CountDownLatch(1);
-            internalCluster().getInstance(ClusterService.class, oldMasterNode).addListener(event -> {
-                if (event.state().nodes().getMasterNodeId() == null) {
-                    oldMasterNodeSteppedDown.countDown();
-                }
-            });
-            internalCluster().clearDisruptionScheme();
-            internalCluster().setDisruptionScheme(masterNodeDisruption);
-            logger.info("--> freezing node [{}]", oldMasterNode);
-            masterNodeDisruption.startDisrupting();
-
-            // Wait for majority side to elect a new master
-            assertBusy(() -> {
-                for (final Map.Entry<String, List<Tuple<String, String>>> entry : masters.entrySet()) {
-                    final List<Tuple<String, String>> transitions = entry.getValue();
-                    assertTrue(entry.getKey() + ": " + transitions, transitions.stream().anyMatch(transition -> transition.v2() != null));
-                }
-            });
-
-            // Save the new elected master node
-            final String newMasterNode = internalCluster().getMasterName(majoritySide.get(0));
-            logger.info("--> new detected master node [{}]", newMasterNode);
-
-            // Stop disruption
-            logger.info("--> unfreezing node [{}]", oldMasterNode);
-            masterNodeDisruption.stopDisrupting();
-
-            oldMasterNodeSteppedDown.await(30, TimeUnit.SECONDS);
-            logger.info("--> [{}] stepped down as master", oldMasterNode);
-            ensureStableCluster(3);
-
-            assertThat(masters.size(), equalTo(2));
-        }
-        List<String> nodeNamesExceptFirstMaster = Arrays.stream(internalCluster().getNodeNames())
-            .filter(name -> name.equals(firstMaster) == false)
-            .toList();
-        /*
-         * It is possible that the first node that became master got re-elected repeatedly. And since it was in a simulated GC when the
-         * other node(s) were master, it only saw itself as master. So we want to check with another node.
-         */
-        Client client = internalCluster().client(randomFrom(nodeNamesExceptFirstMaster));
-        assertMasterStability(client, HealthStatus.YELLOW, containsString(expectedMasterStabilitySymptomSubstring));
-    }
-
-    public void testRepeatedNullMasterRecognizedAsGreenIfMasterDoesNotKnowItIsUnstable() throws Exception {
-        assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20);
-        /*
-         * In this test we have a single master-eligible node. We pause it repeatedly (simulating a long GC pause for example) so that
-         * other nodes decide it is no longer the master. However since there is no other master-eligible node, another node is never
-         * elected master. And the master node never recognizes that it had a problem. So when we run the master stability check on one
-         * of the data nodes, it will see that there is a problem (the master has gone null repeatedly), but when it checks with the
-         * master, the master says everything is fine. So we expect a GREEN status.
-         */
-        final List<String> masterNodes = internalCluster().startMasterOnlyNodes(
-            1,
-            Settings.builder()
-                .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s")
-                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s")
-                .put(CoordinationDiagnosticsService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), 1)
-                .build()
-        );
-        int nullTransitionsThreshold = 1;
-        final List<String> dataNodes = internalCluster().startDataOnlyNodes(
-            2,
-            Settings.builder()
-                .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s")
-                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s")
-                .put(CoordinationDiagnosticsService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), nullTransitionsThreshold)
-                .put(CoordinationDiagnosticsService.NODE_HAS_MASTER_LOOKUP_TIMEFRAME_SETTING.getKey(), new TimeValue(60, TimeUnit.SECONDS))
-                .build()
-        );
-        ensureStableCluster(3);
-        for (int i = 0; i < nullTransitionsThreshold + 1; i++) {
-            final String masterNode = masterNodes.get(0);
-
-            // Simulating a painful gc by suspending all threads for a long time on the current elected master node.
-            SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), masterNode);
-
-            final CountDownLatch dataNodeMasterSteppedDown = new CountDownLatch(2);
-            internalCluster().getInstance(ClusterService.class, dataNodes.get(0)).addListener(event -> {
-                if (event.state().nodes().getMasterNodeId() == null) {
-                    dataNodeMasterSteppedDown.countDown();
-                }
-            });
-            internalCluster().getInstance(ClusterService.class, dataNodes.get(1)).addListener(event -> {
-                if (event.state().nodes().getMasterNodeId() == null) {
-                    dataNodeMasterSteppedDown.countDown();
-                }
-            });
-            internalCluster().clearDisruptionScheme();
-            internalCluster().setDisruptionScheme(masterNodeDisruption);
-            logger.info("--> freezing node [{}]", masterNode);
-            masterNodeDisruption.startDisrupting();
-            dataNodeMasterSteppedDown.await(30, TimeUnit.SECONDS);
-            // Stop disruption
-            logger.info("--> unfreezing node [{}]", masterNode);
-            masterNodeDisruption.stopDisrupting();
-            ensureStableCluster(3, TimeValue.timeValueSeconds(30), false, randomFrom(dataNodes));
-        }
-        assertGreenMasterStability(internalCluster().client(randomFrom(dataNodes)));
-    }
-
     public void testNoMasterEligibleNodes() throws Exception {
         /*
          * In this test we have a single master-eligible node. We then stop the master. We set the master lookup threshold very low on the
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java
index eb10877f5892d..97994a38c277c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java
@@ -152,7 +152,7 @@ public void testIndexActions() throws Exception {
         for (int i = 0; i < 5; i++) {
             // test successful
             assertNoFailuresAndResponse(prepareSearch("test").setSize(0).setQuery(matchAllQuery()), countResponse -> {
-                assertThat(countResponse.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(countResponse.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(countResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
                 assertThat(countResponse.getFailedShards(), equalTo(0));
             });
@@ -164,7 +164,7 @@ public void testIndexActions() throws Exception {
                     countResponse.getShardFailures() == null ? 0 : countResponse.getShardFailures().length,
                     equalTo(0)
                 );
-                assertThat(countResponse.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(countResponse.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(countResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
                 assertThat(countResponse.getFailedShards(), equalTo(0));
             });
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java
index f813932ebe924..ecd5c5af8649f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java
@@ -123,7 +123,7 @@ public Settings onNodeStopped(String nodeName) {
 
     public void testFailsToStartIfDowngraded() {
         final IllegalStateException illegalStateException = expectThrowsOnRestart(
-            dataPaths -> PersistedClusterStateService.overrideVersion(NodeMetadataTests.tooNewVersion(), dataPaths)
+            dataPaths -> PersistedClusterStateService.overrideVersion(NodeMetadataTests.tooNewBuildVersion(), dataPaths)
         );
         assertThat(
             illegalStateException.getMessage(),
@@ -133,7 +133,7 @@ public void testFailsToStartIfDowngraded() {
 
     public void testFailsToStartIfUpgradedTooFar() {
         final IllegalStateException illegalStateException = expectThrowsOnRestart(
-            dataPaths -> PersistedClusterStateService.overrideVersion(NodeMetadataTests.tooOldVersion(), dataPaths)
+            dataPaths -> PersistedClusterStateService.overrideVersion(NodeMetadataTests.tooOldBuildVersion(), dataPaths)
         );
         assertThat(
             illegalStateException.getMessage(),
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index 00bd350fe2b84..cdd5a52e048bd 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.coordination.CoordinationMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
@@ -27,14 +26,9 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.core.IOUtils;
-import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.env.NodeMetadata;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.indices.IndexClosedException;
@@ -46,13 +40,8 @@
 import org.elasticsearch.xcontent.XContentFactory;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -60,7 +49,6 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.notNullValue;
@@ -545,52 +533,4 @@ public void testArchiveBrokenClusterSettings() throws Exception {
         assertHitCount(prepareSearch().setQuery(matchAllQuery()), 1L);
     }
 
-    public void testHalfDeletedIndexImport() throws Exception {
-        // It's possible for a 6.x node to add a tombstone for an index but not actually delete the index metadata from disk since that
-        // deletion is slightly deferred and may race against the node being shut down; if you upgrade to 7.x when in this state then the
-        // node won't start.
-
-        final String nodeName = internalCluster().startNode();
-        createIndex("test", 1, 0);
-        ensureGreen("test");
-
-        final Metadata metadata = internalCluster().getInstance(ClusterService.class).state().metadata();
-        final Path[] paths = internalCluster().getInstance(NodeEnvironment.class).nodeDataPaths();
-        final String nodeId = clusterAdmin().prepareNodesInfo(nodeName).clear().get().getNodes().get(0).getNode().getId();
-
-        writeBrokenMeta(nodeEnvironment -> {
-            for (final Path path : paths) {
-                IOUtils.rm(path.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME));
-            }
-            MetaStateWriterUtils.writeGlobalState(
-                nodeEnvironment,
-                "test",
-                Metadata.builder(metadata)
-                    // we remove the manifest file, resetting the term and making this look like an upgrade from 6.x, so must also reset the
-                    // term in the coordination metadata
-                    .coordinationMetadata(CoordinationMetadata.builder(metadata.coordinationMetadata()).term(0L).build())
-                    // add a tombstone but do not delete the index metadata from disk
-                    .putCustom(IndexGraveyard.TYPE, IndexGraveyard.builder().addTombstone(metadata.index("test").getIndex()).build())
-                    .build()
-            );
-            NodeMetadata.FORMAT.writeAndCleanup(new NodeMetadata(nodeId, BuildVersion.current(), metadata.oldestIndexVersion()), paths);
-        });
-
-        ensureGreen();
-
-        assertBusy(() -> assertThat(internalCluster().getInstance(NodeEnvironment.class).availableIndexFolders(), empty()));
-    }
-
-    private void writeBrokenMeta(CheckedConsumer<NodeEnvironment, IOException> writer) throws Exception {
-        Map<String, NodeEnvironment> nodeEnvironments = Stream.of(internalCluster().getNodeNames())
-            .collect(Collectors.toMap(Function.identity(), nodeName -> internalCluster().getInstance(NodeEnvironment.class, nodeName)));
-        internalCluster().fullRestart(new RestartCallback() {
-            @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                final NodeEnvironment nodeEnvironment = nodeEnvironments.get(nodeName);
-                writer.accept(nodeEnvironment);
-                return super.onNodeStopped(nodeName);
-            }
-        });
-    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java
index 5da9788e3079f..4d1ed9bce6440 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java
@@ -115,7 +115,7 @@ public void testFinalPipelineOfOldDestinationIsNotInvoked() {
             .get();
         assertEquals(RestStatus.CREATED, indexResponse.status());
         assertResponse(prepareSearch("target"), response -> {
-            assertEquals(1, response.getHits().getTotalHits().value);
+            assertEquals(1, response.getHits().getTotalHits().value());
             assertFalse(response.getHits().getAt(0).getSourceAsMap().containsKey("final"));
         });
     }
@@ -139,7 +139,7 @@ public void testFinalPipelineOfNewDestinationIsInvoked() {
             .get();
         assertEquals(RestStatus.CREATED, indexResponse.status());
         assertResponse(prepareSearch("target"), response -> {
-            assertEquals(1, response.getHits().getTotalHits().value);
+            assertEquals(1, response.getHits().getTotalHits().value());
             assertEquals(true, response.getHits().getAt(0).getSourceAsMap().get("final"));
         });
     }
@@ -163,7 +163,7 @@ public void testDefaultPipelineOfNewDestinationIsNotInvoked() {
             .get();
         assertEquals(RestStatus.CREATED, indexResponse.status());
         assertResponse(prepareSearch("target"), response -> {
-            assertEquals(1, response.getHits().getTotalHits().value);
+            assertEquals(1, response.getHits().getTotalHits().value());
             assertFalse(response.getHits().getAt(0).getSourceAsMap().containsKey("final"));
         });
     }
@@ -187,7 +187,7 @@ public void testDefaultPipelineOfRerouteDestinationIsInvoked() {
             .get();
         assertEquals(RestStatus.CREATED, indexResponse.status());
         assertResponse(prepareSearch("target"), response -> {
-            assertEquals(1, response.getHits().getTotalHits().value);
+            assertEquals(1, response.getHits().getTotalHits().value());
             assertTrue(response.getHits().getAt(0).getSourceAsMap().containsKey("final"));
         });
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java
new file mode 100644
index 0000000000000..f294d4a2e7943
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.admin.indices.shrink.ResizeAction;
+import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
+import org.elasticsearch.action.admin.indices.shrink.ResizeType;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+public class LookupIndexModeIT extends ESIntegTestCase {
+
+    @Override
+    protected int numberOfShards() {
+        return 1;
+    }
+
+    public void testBasic() {
+        internalCluster().ensureAtLeastNumDataNodes(1);
+        Settings.Builder lookupSettings = Settings.builder().put("index.mode", "lookup");
+        if (randomBoolean()) {
+            lookupSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
+        }
+        CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
+        createRequest.settings(lookupSettings);
+        createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
+        assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
+        Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
+        assertThat(settings.get("index.mode"), equalTo("lookup"));
+        assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
+        Map<String, String> allHosts = Map.of(
+            "192.168.1.2",
+            "Windows",
+            "192.168.1.3",
+            "MacOS",
+            "192.168.1.4",
+            "Linux",
+            "192.168.1.5",
+            "Android",
+            "192.168.1.6",
+            "iOS",
+            "192.168.1.7",
+            "Windows",
+            "192.168.1.8",
+            "MacOS",
+            "192.168.1.9",
+            "Linux",
+            "192.168.1.10",
+            "Linux",
+            "192.168.1.11",
+            "Windows"
+        );
+        for (Map.Entry<String, String> e : allHosts.entrySet()) {
+            client().prepareIndex("hosts").setSource("ip", e.getKey(), "os", e.getValue()).get();
+        }
+        refresh("hosts");
+        assertAcked(client().admin().indices().prepareCreate("events").setSettings(Settings.builder().put("index.mode", "logsdb")).get());
+        int numDocs = between(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            String ip = randomFrom(allHosts.keySet());
+            String message = randomFrom("login", "logout", "shutdown", "restart");
+            client().prepareIndex("events").setSource("@timestamp", "2024-01-01", "ip", ip, "message", message).get();
+        }
+        refresh("events");
+        // _search
+        {
+            SearchResponse resp = prepareSearch("events", "hosts").setQuery(new MatchQueryBuilder("_index_mode", "lookup"))
+                .setSize(10000)
+                .get();
+            for (SearchHit hit : resp.getHits()) {
+                assertThat(hit.getIndex(), equalTo("hosts"));
+            }
+            assertHitCount(resp, allHosts.size());
+            resp.decRef();
+        }
+        // field_caps
+        {
+            FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+            request.indices("events", "hosts");
+            request.fields("*");
+            request.setMergeResults(false);
+            request.indexFilter(new MatchQueryBuilder("_index_mode", "lookup"));
+            var resp = client().fieldCaps(request).actionGet();
+            assertThat(resp.getIndexResponses(), hasSize(1));
+            FieldCapabilitiesIndexResponse indexResponse = resp.getIndexResponses().getFirst();
+            assertThat(indexResponse.getIndexMode(), equalTo(IndexMode.LOOKUP));
+            assertThat(indexResponse.getIndexName(), equalTo("hosts"));
+        }
+    }
+
+    public void testRejectMoreThanOneShard() {
+        int numberOfShards = between(2, 5);
+        IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> {
+            client().admin()
+                .indices()
+                .prepareCreate("hosts")
+                .setSettings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))
+                .setMapping("ip", "type=ip", "os", "type=keyword")
+                .get();
+        });
+        assertThat(
+            error.getMessage(),
+            equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided " + numberOfShards)
+        );
+    }
+
+    public void testResizeLookupIndex() {
+        Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
+        if (randomBoolean()) {
+            createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
+        }
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest("lookup-1").settings(createSettings);
+        assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createIndexRequest));
+        client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "lookup-1").get();
+
+        ResizeRequest clone = new ResizeRequest("lookup-2", "lookup-1");
+        clone.setResizeType(ResizeType.CLONE);
+        assertAcked(client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet());
+        Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2");
+        assertThat(settings.get("index.mode"), equalTo("lookup"));
+        assertThat(settings.get("index.number_of_shards"), equalTo("1"));
+        assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
+
+        ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1");
+        split.setResizeType(ResizeType.SPLIT);
+        split.getTargetIndexRequest().settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3));
+        IllegalArgumentException error = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().execute(ResizeAction.INSTANCE, split).actionGet()
+        );
+        assertThat(
+            error.getMessage(),
+            equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 3")
+        );
+    }
+
+    public void testResizeRegularIndexToLookup() {
+        String dataNode = internalCluster().startDataOnlyNode();
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareCreate("regular-1")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
+                        .put("index.routing.allocation.require._name", dataNode)
+                )
+                .setMapping("ip", "type=ip", "os", "type=keyword")
+                .get()
+        );
+        client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "regular-1").get();
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("regular-1")
+            .setSettings(Settings.builder().put("index.number_of_replicas", 0))
+            .get();
+
+        ResizeRequest clone = new ResizeRequest("lookup-3", "regular-1");
+        clone.setResizeType(ResizeType.CLONE);
+        clone.getTargetIndexRequest().settings(Settings.builder().put("index.mode", "lookup"));
+        IllegalArgumentException error = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet()
+        );
+        assertThat(
+            error.getMessage(),
+            equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 2")
+        );
+
+        ResizeRequest shrink = new ResizeRequest("lookup-4", "regular-1");
+        shrink.setResizeType(ResizeType.SHRINK);
+        shrink.getTargetIndexRequest()
+            .settings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1));
+
+        error = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().execute(ResizeAction.INSTANCE, shrink).actionGet()
+        );
+        assertThat(error.getMessage(), equalTo("can't change index.mode of index [regular-1] from [standard] to [lookup]"));
+    }
+
+    public void testDoNotOverrideAutoExpandReplicas() {
+        internalCluster().ensureAtLeastNumDataNodes(1);
+        Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
+        if (randomBoolean()) {
+            createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
+        }
+        createSettings.put("index.auto_expand_replicas", "3-5");
+        CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
+        createRequest.settings(createSettings);
+        createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
+        assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
+        Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
+        assertThat(settings.get("index.mode"), equalTo("lookup"));
+        assertThat(settings.get("index.auto_expand_replicas"), equalTo("3-5"));
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java
index 7b7433e3aa4c3..cb280d5577fae 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java
@@ -107,7 +107,7 @@ public void testMaxDocsLimit() throws Exception {
         indicesAdmin().prepareRefresh("test").get();
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(new MatchAllQueryBuilder()).setTrackTotalHitsUpTo(Integer.MAX_VALUE).setSize(0),
-            response -> assertThat(response.getHits().getTotalHits().value, equalTo((long) maxDocs.get()))
+            response -> assertThat(response.getHits().getTotalHits().value(), equalTo((long) maxDocs.get()))
         );
         if (randomBoolean()) {
             indicesAdmin().prepareFlush("test").get();
@@ -117,7 +117,7 @@ public void testMaxDocsLimit() throws Exception {
         ensureGreen("test");
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(new MatchAllQueryBuilder()).setTrackTotalHitsUpTo(Integer.MAX_VALUE).setSize(0),
-            response -> assertThat(response.getHits().getTotalHits().value, equalTo((long) maxDocs.get()))
+            response -> assertThat(response.getHits().getTotalHits().value(), equalTo((long) maxDocs.get()))
         );
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java
index 81a0e0ede7cd3..1194218c68ff1 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java
@@ -46,7 +46,7 @@ public void testDynamicTemplateCopyTo() throws Exception {
                     AggregationBuilders.terms("test_raw").field("test_field_raw").size(recordCount * 2).collectMode(aggCollectionMode)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo((long) recordCount));
+                assertThat(response.getHits().getTotalHits().value(), equalTo((long) recordCount));
 
                 assertThat(((Terms) response.getAggregations().get("test")).getBuckets().size(), equalTo(recordCount + 1));
                 assertThat(((Terms) response.getAggregations().get("test_raw")).getBuckets().size(), equalTo(recordCount));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java
index 03afabaae1d0d..902dd911ddcd3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java
@@ -115,7 +115,7 @@ public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, I
                         assertResponse(
                             prepareSearch("index").setQuery(termQuery("_id", response.getHits().getHits()[i].getId())).setExplain(true),
                             dupIdResponse -> {
-                                assertThat(dupIdResponse.getHits().getTotalHits().value, greaterThan(1L));
+                                assertThat(dupIdResponse.getHits().getTotalHits().value(), greaterThan(1L));
                                 logger.info("found a duplicate id:");
                                 for (SearchHit hit : dupIdResponse.getHits()) {
                                     logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java
index 62c5f934ec8b6..37fbc95d56506 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java
@@ -57,9 +57,9 @@ public void testAutoGenerateIdNoDuplicates() throws Exception {
                 try {
                     logger.debug("running search with all types");
                     assertResponse(prepareSearch("test"), response -> {
-                        if (response.getHits().getTotalHits().value != numOfDocs) {
+                        if (response.getHits().getTotalHits().value() != numOfDocs) {
                             final String message = "Count is "
-                                + response.getHits().getTotalHits().value
+                                + response.getHits().getTotalHits().value()
                                 + " but "
                                 + numOfDocs
                                 + " was expected. "
@@ -77,9 +77,9 @@ public void testAutoGenerateIdNoDuplicates() throws Exception {
                 try {
                     logger.debug("running search with a specific type");
                     assertResponse(prepareSearch("test"), response -> {
-                        if (response.getHits().getTotalHits().value != numOfDocs) {
+                        if (response.getHits().getTotalHits().value() != numOfDocs) {
                             final String message = "Count is "
-                                + response.getHits().getTotalHits().value
+                                + response.getHits().getTotalHits().value()
                                 + " but "
                                 + numOfDocs
                                 + " was expected. "
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index 7db810fc70ac1..52492ba7ce657 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -149,7 +149,7 @@ public void testQueryRewrite() throws Exception {
                 .addAggregation(new GlobalAggregationBuilder("global")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 0, 5);
@@ -161,7 +161,7 @@ public void testQueryRewrite() throws Exception {
                 .addAggregation(new GlobalAggregationBuilder("global")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
 
             }
         );
@@ -174,7 +174,7 @@ public void testQueryRewrite() throws Exception {
                 .addAggregation(new GlobalAggregationBuilder("global")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 6, 9);
@@ -217,7 +217,7 @@ public void testQueryRewriteMissingValues() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
             }
         );
         assertCacheState(client, "index", 0, 1);
@@ -229,7 +229,7 @@ public void testQueryRewriteMissingValues() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
             }
         );
         assertCacheState(client, "index", 1, 1);
@@ -241,7 +241,7 @@ public void testQueryRewriteMissingValues() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
             }
         );
         assertCacheState(client, "index", 2, 1);
@@ -286,7 +286,7 @@ public void testQueryRewriteDates() throws Exception {
                 .addAggregation(new GlobalAggregationBuilder("global")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(9L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(9L));
             }
         );
         assertCacheState(client, "index", 0, 1);
@@ -299,7 +299,7 @@ public void testQueryRewriteDates() throws Exception {
                 .addAggregation(new GlobalAggregationBuilder("global")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(9L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(9L));
             }
         );
         assertCacheState(client, "index", 1, 1);
@@ -312,7 +312,7 @@ public void testQueryRewriteDates() throws Exception {
                 .addAggregation(new GlobalAggregationBuilder("global")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(9L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(9L));
             }
         );
         assertCacheState(client, "index", 2, 1);
@@ -364,7 +364,7 @@ public void testQueryRewriteDatesWithNow() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
             }
         );
         assertCacheState(client, "index-1", 0, 1);
@@ -381,7 +381,7 @@ public void testQueryRewriteDatesWithNow() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
             }
         );
         assertCacheState(client, "index-1", 1, 1);
@@ -395,7 +395,7 @@ public void testQueryRewriteDatesWithNow() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
             }
         );
         assertCacheState(client, "index-1", 2, 1);
@@ -440,7 +440,7 @@ public void testCanCache() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 0, 0);
@@ -453,7 +453,7 @@ public void testCanCache() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 0, 0);
@@ -468,7 +468,7 @@ public void testCanCache() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 0, 0);
@@ -483,7 +483,7 @@ public void testCanCache() throws Exception {
                 .addAggregation(dateRange("foo").field("s").addRange("now-10y", "now")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 0, 0);
@@ -497,7 +497,7 @@ public void testCanCache() throws Exception {
                 .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 0, 2);
@@ -512,7 +512,7 @@ public void testCanCache() throws Exception {
                 .addAggregation(filter("foo", QueryBuilders.rangeQuery("s").from("now-10y").to("now"))),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(7L));
             }
         );
         assertCacheState(client, "index", 0, 4);
@@ -543,7 +543,7 @@ public void testCacheWithFilteredAlias() {
                 .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             }
         );
         assertCacheState(client, "index", 0, 1);
@@ -555,20 +555,20 @@ public void testCacheWithFilteredAlias() {
                 .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")),
             response -> {
                 ElasticsearchAssertions.assertAllSuccessful(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             }
         );
         assertCacheState(client, "index", 1, 1);
 
         assertResponse(client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0), response -> {
             ElasticsearchAssertions.assertAllSuccessful(response);
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
         });
         assertCacheState(client, "index", 1, 2);
 
         assertResponse(client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0), response -> {
             ElasticsearchAssertions.assertAllSuccessful(response);
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
         });
         assertCacheState(client, "index", 2, 2);
     }
@@ -591,7 +591,7 @@ public void testProfileDisableCache() throws Exception {
                 client.prepareSearch("index").setRequestCache(true).setProfile(profile).setQuery(QueryBuilders.termQuery("k", "hello")),
                 response -> {
                     ElasticsearchAssertions.assertAllSuccessful(response);
-                    assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 }
             );
             if (profile == false) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
index a6b168af5268d..cbb0a67edcb83 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
@@ -229,7 +229,7 @@ public void testCloseWhileRelocatingShards() throws Exception {
 
             for (String index : acknowledgedCloses) {
                 assertResponse(prepareSearch(index).setSize(0).setTrackTotalHits(true), response -> {
-                    long docsCount = response.getHits().getTotalHits().value;
+                    long docsCount = response.getHits().getTotalHits().value();
                     assertEquals(
                         "Expected "
                             + docsPerIndex.get(index)
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 3ca3f20917009..0647a24aa39c8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -881,7 +881,7 @@ public void testPartitionedTemplate() throws Exception {
         );
         assertThat(
             eBadSettings.getMessage(),
-            containsString("partition size [6] should be a positive number less than the number of shards [5]")
+            containsString("partition size [6] should be a positive number less than the number of routing shards [5]")
         );
 
         // provide an invalid mapping for a partitioned index
@@ -913,7 +913,7 @@ public void testPartitionedTemplate() throws Exception {
 
         assertThat(
             eBadIndex.getMessage(),
-            containsString("partition size [6] should be a positive number less than the number of shards [5]")
+            containsString("partition size [6] should be a positive number less than the number of routing shards [5]")
         );
 
         // finally, create a valid index
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
index 942f86017c617..77c4f8a26f478 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
@@ -344,7 +344,7 @@ private void iterateAssertCount(final int numberOfShards, final int iterations,
                 prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).setTrackTotalHits(true).addSort("id", SortOrder.ASC),
                 response -> {
                     logSearchResponse(numberOfShards, numberOfDocs, finalI, response);
-                    iterationHitCount[finalI] = response.getHits().getTotalHits().value;
+                    iterationHitCount[finalI] = response.getHits().getTotalHits().value();
                     if (iterationHitCount[finalI] != numberOfDocs) {
                         error[0] = true;
                     }
@@ -391,7 +391,7 @@ private void iterateAssertCount(final int numberOfShards, final int iterations,
                 boolean[] errorOccurred = new boolean[1];
                 for (int i = 0; i < iterations; i++) {
                     assertResponse(prepareSearch().setTrackTotalHits(true).setSize(0).setQuery(matchAllQuery()), response -> {
-                        if (response.getHits().getTotalHits().value != numberOfDocs) {
+                        if (response.getHits().getTotalHits().value() != numberOfDocs) {
                             errorOccurred[0] = true;
                         }
                     });
@@ -421,7 +421,7 @@ private void logSearchResponse(int numberOfShards, long numberOfDocs, int iterat
         logger.info(
             "iteration [{}] - returned documents: {} (expected {})",
             iteration,
-            searchResponse.getHits().getTotalHits().value,
+            searchResponse.getHits().getTotalHits().value(),
             numberOfDocs
         );
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java
index fb1fabfd198e6..2c56f75b051eb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java
@@ -240,7 +240,7 @@ public void testRelocationWhileIndexingRandom() throws Exception {
                     prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).storedFields(),
                     response -> {
                         var hits = response.getHits();
-                        if (hits.getTotalHits().value != indexer.totalIndexedDocs()) {
+                        if (hits.getTotalHits().value() != indexer.totalIndexedDocs()) {
                             int[] hitIds = new int[(int) indexer.totalIndexedDocs()];
                             for (int hit = 0; hit < indexer.totalIndexedDocs(); hit++) {
                                 hitIds[hit] = hit + 1;
@@ -254,7 +254,7 @@ public void testRelocationWhileIndexingRandom() throws Exception {
                             }
                             set.forEach(value -> logger.error("Missing id [{}]", value));
                         }
-                        assertThat(hits.getTotalHits().value, equalTo(indexer.totalIndexedDocs()));
+                        assertThat(hits.getTotalHits().value(), equalTo(indexer.totalIndexedDocs()));
                         logger.info("--> DONE search test round {}", idx + 1);
                     }
                 );
@@ -364,9 +364,9 @@ public void indexShardStateChanged(
             for (Client client : clients()) {
                 assertNoFailuresAndResponse(client.prepareSearch("test").setPreference("_local").setSize(0), response -> {
                     if (expectedCount[0] < 0) {
-                        expectedCount[0] = response.getHits().getTotalHits().value;
+                        expectedCount[0] = response.getHits().getTotalHits().value();
                     } else {
-                        assertEquals(expectedCount[0], response.getHits().getTotalHits().value);
+                        assertEquals(expectedCount[0], response.getHits().getTotalHits().value());
                     }
                 });
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
index c618e354802a7..f9122ccfb4a3e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
 
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -40,6 +41,7 @@
 import static org.elasticsearch.test.NodeRoles.dataOnlyNode;
 import static org.elasticsearch.test.NodeRoles.masterNode;
 import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -50,7 +52,12 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class FileSettingsServiceIT extends ESIntegTestCase {
 
-    private static final AtomicLong versionCounter = new AtomicLong(1);
+    private final AtomicLong versionCounter = new AtomicLong(1);
+
+    @Before
+    public void resetVersionCounter() {
+        versionCounter.set(1);
+    }
 
     private static final String testJSON = """
         {
@@ -102,6 +109,19 @@ public class FileSettingsServiceIT extends ESIntegTestCase {
              }
         }""";
 
+    private static final String testOtherErrorJSON = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "bad_cluster_settings": {
+                     "search.allow_expensive_queries": "false"
+                 }
+             }
+        }""";
+
     private void assertMasterNode(Client client, String node) {
         assertThat(
             client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName(),
@@ -109,8 +129,9 @@ private void assertMasterNode(Client client, String node) {
         );
     }
 
-    public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger) throws Exception {
-        long version = versionCounter.incrementAndGet();
+    public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger, boolean incrementVersion)
+        throws Exception {
+        long version = incrementVersion ? versionCounter.incrementAndGet() : versionCounter.get();
 
         FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
 
@@ -124,6 +145,15 @@ public static void writeJSONFile(String node, String json, AtomicLong versionCou
         logger.info("--> After writing new settings file: [{}]", settingsFileContent);
     }
 
+    public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger) throws Exception {
+        writeJSONFile(node, json, versionCounter, logger, true);
+    }
+
+    public static void writeJSONFileWithoutVersionIncrement(String node, String json, AtomicLong versionCounter, Logger logger)
+        throws Exception {
+        writeJSONFile(node, json, versionCounter, logger, false);
+    }
+
     private Tuple<CountDownLatch, AtomicLong> setupCleanupClusterStateListener(String node) {
         ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
@@ -171,7 +201,10 @@ public void clusterChanged(ClusterChangedEvent event) {
     private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion, String expectedBytesPerSec)
         throws Exception {
         assertTrue(savedClusterState.await(20, TimeUnit.SECONDS));
+        assertExpectedRecoveryBytesSettingAndVersion(metadataVersion, expectedBytesPerSec);
+    }
 
+    private static void assertExpectedRecoveryBytesSettingAndVersion(AtomicLong metadataVersion, String expectedBytesPerSec) {
         final ClusterStateResponse clusterStateResponse = clusterAdmin().state(
             new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(metadataVersion.get())
         ).actionGet();
@@ -337,6 +370,77 @@ public void testErrorSaved() throws Exception {
         assertClusterStateNotSaved(savedClusterState.v1(), savedClusterState.v2());
     }
 
+    public void testErrorCanRecoverOnRestart() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        logger.info("--> start data node / non master node");
+        String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
+        FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode);
+
+        assertFalse(dataFileSettingsService.watching());
+
+        logger.info("--> start master node");
+        final String masterNode = internalCluster().startMasterOnlyNode(
+            Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
+        );
+        assertMasterNode(internalCluster().nonMasterClient(), masterNode);
+        var savedClusterState = setupClusterStateListenerForError(masterNode);
+
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        assertTrue(masterFileSettingsService.watching());
+        assertFalse(dataFileSettingsService.watching());
+
+        writeJSONFile(masterNode, testErrorJSON, versionCounter, logger);
+        AtomicLong metadataVersion = savedClusterState.v2();
+        assertClusterStateNotSaved(savedClusterState.v1(), metadataVersion);
+        assertHasErrors(metadataVersion, "not_cluster_settings");
+
+        // write valid json without version increment to simulate ES being able to process settings after a restart (usually, this would be
+        // due to a code change)
+        writeJSONFileWithoutVersionIncrement(masterNode, testJSON, versionCounter, logger);
+        internalCluster().restartNode(masterNode);
+        ensureGreen();
+
+        // we don't know the exact metadata version to wait for so rely on an assertBusy instead
+        assertBusy(() -> assertExpectedRecoveryBytesSettingAndVersion(metadataVersion, "50mb"));
+        assertBusy(() -> assertNoErrors(metadataVersion));
+    }
+
+    public void testNewErrorOnRestartReprocessing() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        logger.info("--> start data node / non master node");
+        String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
+        FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode);
+
+        assertFalse(dataFileSettingsService.watching());
+
+        logger.info("--> start master node");
+        final String masterNode = internalCluster().startMasterOnlyNode(
+            Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
+        );
+        assertMasterNode(internalCluster().nonMasterClient(), masterNode);
+        var savedClusterState = setupClusterStateListenerForError(masterNode);
+
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        assertTrue(masterFileSettingsService.watching());
+        assertFalse(dataFileSettingsService.watching());
+
+        writeJSONFile(masterNode, testErrorJSON, versionCounter, logger);
+        AtomicLong metadataVersion = savedClusterState.v2();
+        assertClusterStateNotSaved(savedClusterState.v1(), metadataVersion);
+        assertHasErrors(metadataVersion, "not_cluster_settings");
+
+        // write json with new error without version increment to simulate ES failing to process settings after a restart for a new reason
+        // (usually, this would be due to a code change)
+        writeJSONFileWithoutVersionIncrement(masterNode, testOtherErrorJSON, versionCounter, logger);
+        assertHasErrors(metadataVersion, "not_cluster_settings");
+        internalCluster().restartNode(masterNode);
+        ensureGreen();
+
+        assertBusy(() -> assertHasErrors(metadataVersion, "bad_cluster_settings"));
+    }
+
     public void testSettingsAppliedOnMasterReElection() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
         logger.info("--> start master node");
@@ -383,4 +487,21 @@ public void testSettingsAppliedOnMasterReElection() throws Exception {
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "43mb");
     }
 
+    private void assertHasErrors(AtomicLong waitForMetadataVersion, String expectedError) {
+        var errorMetadata = getErrorMetadata(waitForMetadataVersion);
+        assertThat(errorMetadata, is(notNullValue()));
+        assertThat(errorMetadata.errors(), containsInAnyOrder(containsString(expectedError)));
+    }
+
+    private void assertNoErrors(AtomicLong waitForMetadataVersion) {
+        var errorMetadata = getErrorMetadata(waitForMetadataVersion);
+        assertThat(errorMetadata, is(nullValue()));
+    }
+
+    private ReservedStateErrorMetadata getErrorMetadata(AtomicLong waitForMetadataVersion) {
+        final ClusterStateResponse clusterStateResponse = clusterAdmin().state(
+            new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(waitForMetadataVersion.get())
+        ).actionGet();
+        return clusterStateResponse.getState().getMetadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE).errorMetadata();
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java
index 45dce5789b9bc..199c9a9fb4c8c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java
@@ -296,7 +296,7 @@ public void testAliasSearchRoutingWithConcreteAndAliasedIndices_issue3268() thro
             prepareSearch("index_*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1).setQuery(QueryBuilders.matchAllQuery()),
             response -> {
                 logger.info("--> search all on index_* should find two");
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 // Let's make sure that, even though 2 docs are available, only one is returned according to the size we set in the request
                 // Therefore the reduce phase has taken place, which proves that the QUERY_AND_FETCH search type wasn't erroneously forced.
                 assertThat(response.getHits().getHits().length, equalTo(1));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java
index 7bccf3db1284e..68bc6656cec7f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java
@@ -160,7 +160,7 @@ private void verifyRoutedSearches(String index, Map<String, Set<String>> routing
                             + "] shards for routing ["
                             + routing
                             + "] and got hits ["
-                            + response.getHits().getTotalHits().value
+                            + response.getHits().getTotalHits().value()
                             + "]"
                     );
 
@@ -168,7 +168,7 @@ private void verifyRoutedSearches(String index, Map<String, Set<String>> routing
                         response.getTotalShards() + " was not in " + expectedShards + " for " + index,
                         expectedShards.contains(response.getTotalShards())
                     );
-                    assertEquals(expectedDocuments, response.getHits().getTotalHits().value);
+                    assertEquals(expectedDocuments, response.getHits().getTotalHits().value());
 
                     Set<String> found = new HashSet<>();
                     response.getHits().forEach(h -> found.add(h.getId()));
@@ -188,7 +188,7 @@ private void verifyBroadSearches(String index, Map<String, Set<String>> routingT
                 prepareSearch().setQuery(QueryBuilders.termQuery("_routing", routing)).setIndices(index).setSize(100),
                 response -> {
                     assertEquals(expectedShards, response.getTotalShards());
-                    assertEquals(expectedDocuments, response.getHits().getTotalHits().value);
+                    assertEquals(expectedDocuments, response.getHits().getTotalHits().value());
 
                     Set<String> found = new HashSet<>();
                     response.getHits().forEach(h -> found.add(h.getId()));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java
index ee1aac60da9c1..f63f09764621b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java
@@ -64,7 +64,7 @@ public void testTopHitsTimeout() {
         assertEquals(0, searchResponse.getFailedShards());
         assertThat(searchResponse.getSuccessfulShards(), greaterThan(0));
         assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards());
-        assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+        assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
         assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
     }
 
@@ -81,7 +81,7 @@ public void testAggsTimeout() {
         assertEquals(0, searchResponse.getFailedShards());
         assertThat(searchResponse.getSuccessfulShards(), greaterThan(0));
         assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards());
-        assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+        assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
         assertEquals(searchResponse.getHits().getHits().length, 0);
         StringTerms terms = searchResponse.getAggregations().get("terms");
         assertEquals(1, terms.getBuckets().size());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java
index d023c9de87ca5..4a407ae66f7ad 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java
@@ -115,7 +115,7 @@ public void testSubAggregationForTopAggregationOnUnmappedField() throws Exceptio
                 histogram("values").field("value1").interval(1).subAggregation(terms("names").field("name").collectMode(aggCollectionMode))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, Matchers.equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), Matchers.equalTo(0L));
                 Histogram values = response.getAggregations().get("values");
                 assertThat(values, notNullValue());
                 assertThat(values.getBuckets().isEmpty(), is(true));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
index 5a21b600cacd4..1a6e1519d4402 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
@@ -293,7 +293,7 @@ public void testDuelTerms() throws Exception {
                 ),
             response -> {
                 assertAllSuccessful(response);
-                assertEquals(numDocs, response.getHits().getTotalHits().value);
+                assertEquals(numDocs, response.getHits().getTotalHits().value());
 
                 final Terms longTerms = response.getAggregations().get("long");
                 final Terms doubleTerms = response.getAggregations().get("double");
@@ -413,7 +413,7 @@ public void testLargeNumbersOfPercentileBuckets() throws Exception {
             ),
             response -> {
                 assertAllSuccessful(response);
-                assertEquals(numDocs, response.getHits().getTotalHits().value);
+                assertEquals(numDocs, response.getHits().getTotalHits().value());
             }
         );
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java
index a820e6e8d1747..2bd19c9d32d44 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java
@@ -57,7 +57,7 @@ public void testWrapperQueryIsRewritten() throws IOException {
         metadata.put(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
         builder.setMetadata(metadata);
         assertResponse(client().prepareSearch("test").setSize(0).addAggregation(builder), response -> {
-            assertEquals(3, response.getHits().getTotalHits().value);
+            assertEquals(3, response.getHits().getTotalHits().value());
             InternalFilters filters = response.getAggregations().get("titles");
             assertEquals(1, filters.getBuckets().size());
             assertEquals(2, filters.getBuckets().get(0).getDocCount());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index 1787b4f784574..c4560c1b00079 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -88,11 +88,11 @@ private static String format(ZonedDateTime date, String pattern) {
     private IndexRequestBuilder indexDoc(String idx, ZonedDateTime date, int value) throws Exception {
         return prepareIndex(idx).setSource(
             jsonBuilder().startObject()
-                .timeField("date", date)
+                .timestampField("date", date)
                 .field("value", value)
                 .startArray("dates")
-                .timeValue(date)
-                .timeValue(date.plusMonths(1).plusDays(1))
+                .timestampValue(date)
+                .timestampValue(date.plusMonths(1).plusDays(1))
                 .endArray()
                 .endObject()
         );
@@ -103,10 +103,10 @@ private IndexRequestBuilder indexDoc(int month, int day, int value) throws Excep
             jsonBuilder().startObject()
                 .field("value", value)
                 .field("constant", 1)
-                .timeField("date", date(month, day))
+                .timestampField("date", date(month, day))
                 .startArray("dates")
-                .timeValue(date(month, day))
-                .timeValue(date(month + 1, day + 1))
+                .timestampValue(date(month, day))
+                .timestampValue(date(month + 1, day + 1))
                 .endArray()
                 .endObject()
         );
@@ -162,53 +162,53 @@ private void getMultiSortDocs(List<IndexRequestBuilder> builders) throws IOExcep
         for (int i = 1; i <= 3; i++) {
             builders.add(
                 prepareIndex("sort_idx").setSource(
-                    jsonBuilder().startObject().timeField("date", date(1, 1)).field("l", 1).field("d", i).endObject()
+                    jsonBuilder().startObject().timestampField("date", date(1, 1)).field("l", 1).field("d", i).endObject()
                 )
             );
             builders.add(
                 prepareIndex("sort_idx").setSource(
-                    jsonBuilder().startObject().timeField("date", date(1, 2)).field("l", 2).field("d", i).endObject()
+                    jsonBuilder().startObject().timestampField("date", date(1, 2)).field("l", 2).field("d", i).endObject()
                 )
             );
         }
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 3)).field("l", 3).field("d", 1).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 3)).field("l", 3).field("d", 1).endObject()
             )
         );
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 3).plusHours(1)).field("l", 3).field("d", 2).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 3).plusHours(1)).field("l", 3).field("d", 2).endObject()
             )
         );
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 4)).field("l", 3).field("d", 1).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 4)).field("l", 3).field("d", 1).endObject()
             )
         );
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 4).plusHours(2)).field("l", 3).field("d", 3).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 4).plusHours(2)).field("l", 3).field("d", 3).endObject()
             )
         );
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 5)).field("l", 5).field("d", 1).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 5)).field("l", 5).field("d", 1).endObject()
             )
         );
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 5).plusHours(12)).field("l", 5).field("d", 2).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 5).plusHours(12)).field("l", 5).field("d", 2).endObject()
             )
         );
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 6)).field("l", 5).field("d", 1).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 6)).field("l", 5).field("d", 1).endObject()
             )
         );
         builders.add(
             prepareIndex("sort_idx").setSource(
-                jsonBuilder().startObject().timeField("date", date(1, 7)).field("l", 5).field("d", 1).endObject()
+                jsonBuilder().startObject().timestampField("date", date(1, 7)).field("l", 5).field("d", 1).endObject()
             )
         );
     }
@@ -974,7 +974,7 @@ public void testEmptyAggregation() throws Exception {
                         .subAggregation(dateHistogram("date_histo").field("value").fixedInterval(DateHistogramInterval.HOUR))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 List<? extends Bucket> buckets = histo.getBuckets();
@@ -997,7 +997,7 @@ public void testSingleValueWithTimeZone() throws Exception {
         IndexRequestBuilder[] reqs = new IndexRequestBuilder[5];
         ZonedDateTime date = date("2014-03-11T00:00:00+00:00");
         for (int i = 0; i < reqs.length; i++) {
-            reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().timeField("date", date).endObject());
+            reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().timestampField("date", date).endObject());
             date = date.plusHours(1);
         }
         indexRandom(true, reqs);
@@ -1011,7 +1011,7 @@ public void testSingleValueWithTimeZone() throws Exception {
                         .format("yyyy-MM-dd:HH-mm-ssZZZZZ")
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(5L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(5L));
 
                 Histogram histo = response.getAggregations().get("date_histo");
                 List<? extends Bucket> buckets = histo.getBuckets();
@@ -1175,7 +1175,7 @@ public void testSingleValueFieldWithExtendedBoundsTimezone() throws Exception {
 
                 assertThat(
                     "Expected 24 buckets for one day aggregation with hourly interval",
-                    response.getHits().getTotalHits().value,
+                    response.getHits().getTotalHits().value(),
                     equalTo(2L)
                 );
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java
index 0afc479474814..21b36391781b8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java
@@ -63,7 +63,7 @@ private void prepareIndex(ZonedDateTime date, int numHours, int stepSizeHours, i
         IndexRequestBuilder[] reqs = new IndexRequestBuilder[numHours];
         for (int i = idxIdStart; i < idxIdStart + reqs.length; i++) {
             reqs[i - idxIdStart] = prepareIndex("idx2").setId("" + i)
-                .setSource(jsonBuilder().startObject().timeField("date", date).endObject());
+                .setSource(jsonBuilder().startObject().timestampField("date", date).endObject());
             date = date.plusHours(stepSizeHours);
         }
         indexRandom(true, reqs);
@@ -78,7 +78,7 @@ public void testSingleValueWithPositiveOffset() throws Exception {
                     dateHistogram("date_histo").field("date").offset("2h").format(DATE_FORMAT).fixedInterval(DateHistogramInterval.DAY)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(5L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(5L));
 
                 Histogram histo = response.getAggregations().get("date_histo");
                 List<? extends Histogram.Bucket> buckets = histo.getBuckets();
@@ -99,7 +99,7 @@ public void testSingleValueWithNegativeOffset() throws Exception {
                     dateHistogram("date_histo").field("date").offset("-2h").format(DATE_FORMAT).fixedInterval(DateHistogramInterval.DAY)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(5L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(5L));
 
                 Histogram histo = response.getAggregations().get("date_histo");
                 List<? extends Histogram.Bucket> buckets = histo.getBuckets();
@@ -128,7 +128,7 @@ public void testSingleValueWithOffsetMinDocCount() throws Exception {
                         .fixedInterval(DateHistogramInterval.DAY)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(24L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(24L));
 
                 Histogram histo = response.getAggregations().get("date_histo");
                 List<? extends Histogram.Bucket> buckets = histo.getBuckets();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
index 6e9a9305eaf4e..9ec459ee565e5 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
@@ -58,10 +58,10 @@ private static IndexRequestBuilder indexDoc(int month, int day, int value) throw
         return prepareIndex("idx").setSource(
             jsonBuilder().startObject()
                 .field("value", value)
-                .timeField("date", date(month, day))
+                .timestampField("date", date(month, day))
                 .startArray("dates")
-                .timeValue(date(month, day))
-                .timeValue(date(month + 1, day + 1))
+                .timestampValue(date(month, day))
+                .timestampValue(date(month + 1, day + 1))
                 .endArray()
                 .endObject()
         );
@@ -578,7 +578,7 @@ public void testEmptyAggregation() throws Exception {
                         .subAggregation(dateRange("date_range").field("value").addRange("0-1", 0, 1))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
@@ -620,8 +620,8 @@ public void testScriptCaching() throws Exception {
         );
         indexRandom(
             true,
-            prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().timeField("date", date(1, 1)).endObject()),
-            prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().timeField("date", date(2, 1)).endObject())
+            prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().timestampField("date", date(1, 1)).endObject()),
+            prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().timestampField("date", date(2, 1)).endObject())
         );
 
         // Make sure we are starting with a clear cache
@@ -722,7 +722,7 @@ public void testRangeWithFormatStringValue() throws Exception {
             prepareSearch(indexName).setSize(0)
                 .addAggregation(dateRange("date_range").field("date").addRange("00:16:40", "00:50:00").addRange("00:50:00", "01:06:40")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Range.Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "00:16:40-00:50:00", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "00:50:00-01:06:40", 3000000L, 4000000L);
@@ -739,7 +739,7 @@ public void testRangeWithFormatStringValue() throws Exception {
                         .format("HH.mm.ss")
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Range.Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "00.16.40-00.50.00", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "00.50.00-01.06.40", 3000000L, 4000000L);
@@ -753,7 +753,7 @@ public void testRangeWithFormatStringValue() throws Exception {
                     dateRange("date_range").field("date").addRange(1000000, 3000000).addRange(3000000, 4000000).format("epoch_millis")
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Range.Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "1000000-3000000", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "3000000-4000000", 3000000L, 4000000L);
@@ -788,7 +788,7 @@ public void testRangeWithFormatNumericValue() throws Exception {
             prepareSearch(indexName).setSize(0)
                 .addAggregation(dateRange("date_range").field("date").addRange(1000, 3000).addRange(3000, 4000)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "1000-3000", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "3000-4000", 3000000L, 4000000L);
@@ -799,7 +799,7 @@ public void testRangeWithFormatNumericValue() throws Exception {
             prepareSearch(indexName).setSize(0)
                 .addAggregation(dateRange("date_range").field("date").addRange("1000", "3000").addRange("3000", "4000")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Range.Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "1000-3000", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "3000-4000", 3000000L, 4000000L);
@@ -810,7 +810,7 @@ public void testRangeWithFormatNumericValue() throws Exception {
             prepareSearch(indexName).setSize(0)
                 .addAggregation(dateRange("date_range").field("date").addRange(1.0e3, 3000.8123).addRange(3000.8123, 4.0e3)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Range.Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "1000-3000", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "3000-4000", 3000000L, 4000000L);
@@ -827,7 +827,7 @@ public void testRangeWithFormatNumericValue() throws Exception {
                         .format("HH.mm.ss")
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Range.Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "00.16.40-00.50.00", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "00.50.00-01.06.40", 3000000L, 4000000L);
@@ -841,7 +841,7 @@ public void testRangeWithFormatNumericValue() throws Exception {
                     dateRange("date_range").field("date").addRange(1000000, 3000000).addRange(3000000, 4000000).format("epoch_millis")
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 List<Range.Bucket> buckets = checkBuckets(response.getAggregations().get("date_range"), "date_range", 2);
                 assertBucket(buckets.get(0), 2L, "1000000-3000000", 1000000L, 3000000L);
                 assertBucket(buckets.get(1), 1L, "3000000-4000000", 3000000L, 4000000L);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
index 1b70b859426d5..96807ed119866 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
@@ -159,7 +159,7 @@ public void testEmptyAggregation() throws Exception {
                     histogram("histo").field("value").interval(1L).minDocCount(0).subAggregation(filter("filter", matchAllQuery()))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
index b030370215cd3..439583de910c1 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
@@ -247,7 +247,7 @@ public void testEmptyAggregation() throws Exception {
                         .subAggregation(filters("filters", new KeyedFilter("all", matchAllQuery())))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
@@ -455,7 +455,7 @@ public void testEmptyAggregationWithOtherBucket() throws Exception {
                         .subAggregation(filters("filters", new KeyedFilter("foo", matchAllQuery())).otherBucket(true).otherBucketKey("bar"))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
index 843e50a5a7e21..907f943e68422 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
@@ -413,7 +413,7 @@ public void testEmptyAggregation() throws Exception {
                         )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
index 2edd567221bef..ad65e6468b812 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
@@ -915,7 +915,7 @@ public void testEmptyAggregation() throws Exception {
                         .subAggregation(histogram("sub_histo").field(SINGLE_VALUED_FIELD_NAME).interval(1L))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 List<? extends Bucket> buckets = histo.getBuckets();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
index 72f1b0cc56b25..5e7cffcc8ef0d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
@@ -351,7 +351,7 @@ public void testEmptyAggregation() throws Exception {
             prepareSearch("empty_bucket_idx").setQuery(matchAllQuery())
                 .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0).subAggregation(nested("nested", "nested"))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
index 8b63efd92a648..1cfd6e00af7ab 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
@@ -866,7 +866,7 @@ public void testEmptyAggregation() throws Exception {
                         .subAggregation(range("range").field(SINGLE_VALUED_FIELD_NAME).addRange("0-2", 0.0, 2.0))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, Matchers.notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
index 5e2a44285e8fa..29bf8a8a0b45a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
@@ -97,7 +97,7 @@ public void testEmptyAggregation() throws Exception {
                     histogram("histo").field("value").interval(1L).minDocCount(0).subAggregation(extendedStats("stats").field("value"))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
@@ -130,7 +130,7 @@ public void testUnmapped() throws Exception {
         assertResponse(
             prepareSearch("idx_unmapped").setQuery(matchAllQuery()).addAggregation(extendedStats("stats").field("value")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
 
                 ExtendedStats stats = response.getAggregations().get("stats");
                 assertThat(stats, notNullValue());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
index 762bc5bdfaf39..ff4150556c011 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
@@ -112,7 +112,7 @@ public void testEmptyAggregation() throws Exception {
                         )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
@@ -138,7 +138,7 @@ public void testUnmapped() throws Exception {
                         .field("value")
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
 
                 PercentileRanks reversePercentiles = response.getAggregations().get("percentile_ranks");
                 assertThat(reversePercentiles, notNullValue());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
index 12ed0a5c1a8e0..fe6dc7abf66a8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
@@ -116,7 +116,7 @@ public void testEmptyAggregation() throws Exception {
                         )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
@@ -143,7 +143,7 @@ public void testUnmapped() throws Exception {
                         .percentiles(0, 10, 15, 100)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
 
                 Percentiles percentiles = response.getAggregations().get("percentiles");
                 assertThat(percentiles, notNullValue());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
index 52425ae1d9f17..4c8fed2c16ddc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
@@ -358,7 +358,7 @@ public void testMap() {
             prepareSearch("idx").setQuery(matchAllQuery())
                 .addAggregation(scriptedMetric("scripted").mapScript(mapScript).combineScript(combineScript).reduceScript(reduceScript)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -407,7 +407,7 @@ public void testMapWithParams() {
                         .reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -467,7 +467,7 @@ public void testInitMutatesParams() {
                         )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -522,7 +522,7 @@ public void testMapCombineWithParams() {
                     scriptedMetric("scripted").params(params).mapScript(mapScript).combineScript(combineScript).reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -586,7 +586,7 @@ public void testInitMapCombineWithParams() {
                         .reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -655,7 +655,7 @@ public void testInitMapCombineReduceWithParams() {
                         .reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -714,7 +714,7 @@ public void testInitMapCombineReduceGetProperty() throws Exception {
                     )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Global global = response.getAggregations().get("global");
                 assertThat(global, notNullValue());
@@ -773,7 +773,7 @@ public void testMapCombineReduceWithParams() {
                     scriptedMetric("scripted").params(params).mapScript(mapScript).combineScript(combineScript).reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -824,7 +824,7 @@ public void testInitMapReduceWithParams() {
                         .reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -869,7 +869,7 @@ public void testMapReduceWithParams() {
                     scriptedMetric("scripted").params(params).mapScript(mapScript).combineScript(combineScript).reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -928,7 +928,7 @@ public void testInitMapCombineReduceWithParamsAndReduceParams() {
                         .reduceScript(reduceScript)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -964,7 +964,7 @@ public void testInitMapCombineReduceWithParamsStored() {
                         .reduceScript(new Script(ScriptType.STORED, null, "reduceScript_stored", Collections.emptyMap()))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
 
                 Aggregation aggregation = response.getAggregations().get("scripted");
                 assertThat(aggregation, notNullValue());
@@ -1025,7 +1025,7 @@ public void testInitMapCombineReduceWithParamsAsSubAgg() {
                         )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(numDocs));
                 Aggregation aggregation = response.getAggregations().get("histo");
                 assertThat(aggregation, notNullValue());
                 assertThat(aggregation, instanceOf(Histogram.class));
@@ -1099,7 +1099,7 @@ public void testEmptyAggregation() throws Exception {
                         )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
index fbe70ec2a40d6..1169f8bbdbf18 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
@@ -56,7 +56,7 @@ public void testEmptyAggregation() throws Exception {
                 ),
             response -> {
                 assertShardExecutionState(response, 0);
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 2a8be6b4244dd..b3ad5c578e618 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -82,7 +82,7 @@ public void testEmptyAggregation() throws Exception {
             prepareSearch("empty_bucket_idx").setQuery(matchAllQuery())
                 .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0).subAggregation(sum("sum").field("value"))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
index 2877f8882d6d6..d6cceb2013701 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
@@ -105,7 +105,7 @@ public void testEmptyAggregation() throws Exception {
                         .subAggregation(randomCompression(percentileRanks("percentile_ranks", new double[] { 10, 15 }).field("value")))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
@@ -146,7 +146,7 @@ public void testUnmapped() throws Exception {
             prepareSearch("idx_unmapped").setQuery(matchAllQuery())
                 .addAggregation(randomCompression(percentileRanks("percentile_ranks", new double[] { 0, 10, 15, 100 })).field("value")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
 
                 PercentileRanks reversePercentiles = response.getAggregations().get("percentile_ranks");
                 assertThat(reversePercentiles, notNullValue());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
index bbcf7b191fe1b..b4072bcf226ed 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
@@ -111,7 +111,7 @@ public void testEmptyAggregation() throws Exception {
                         .subAggregation(randomCompression(percentiles("percentiles").field("value")).percentiles(10, 15))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 Histogram histo = response.getAggregations().get("histo");
                 assertThat(histo, notNullValue());
                 Histogram.Bucket bucket = histo.getBuckets().get(1);
@@ -132,7 +132,7 @@ public void testUnmapped() throws Exception {
             prepareSearch("idx_unmapped").setQuery(matchAllQuery())
                 .addAggregation(randomCompression(percentiles("percentiles")).field("value").percentiles(0, 10, 15, 100)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
 
                 Percentiles percentiles = response.getAggregations().get("percentiles");
                 assertThat(percentiles, notNullValue());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index 7ac8e3c7a35b4..80c47d6180db0 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -328,7 +328,7 @@ public void testBasics() throws Exception {
                     assertThat(bucket.getDocCount(), equalTo(10L));
                     TopHits topHits = bucket.getAggregations().get("hits");
                     SearchHits hits = topHits.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(10L));
+                    assertThat(hits.getTotalHits().value(), equalTo(10L));
                     assertThat(hits.getHits().length, equalTo(3));
                     higestSortValue += 10;
                     assertThat((Long) hits.getAt(0).getSortValues()[0], equalTo(higestSortValue));
@@ -348,7 +348,7 @@ public void testIssue11119() throws Exception {
                 .setQuery(matchQuery("text", "x y z"))
                 .addAggregation(terms("terms").executionHint(randomExecutionHint()).field("group").subAggregation(topHits("hits"))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
                 assertThat(response.getHits().getHits().length, equalTo(0));
                 assertThat(response.getHits().getMaxScore(), equalTo(Float.NaN));
                 Terms terms = response.getAggregations().get("terms");
@@ -381,7 +381,7 @@ public void testIssue11119() throws Exception {
                 .setQuery(matchQuery("text", "x y z"))
                 .addAggregation(terms("terms").executionHint(randomExecutionHint()).field("group")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(8L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(8L));
                 assertThat(response.getHits().getHits().length, equalTo(0));
                 assertThat(response.getHits().getMaxScore(), equalTo(Float.NaN));
                 Terms terms = response.getAggregations().get("terms");
@@ -413,7 +413,7 @@ public void testBreadthFirstWithScoreNeeded() throws Exception {
                     assertThat(bucket.getDocCount(), equalTo(10L));
                     TopHits topHits = bucket.getAggregations().get("hits");
                     SearchHits hits = topHits.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(10L));
+                    assertThat(hits.getTotalHits().value(), equalTo(10L));
                     assertThat(hits.getHits().length, equalTo(3));
 
                     assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
@@ -444,7 +444,7 @@ public void testBreadthFirstWithAggOrderAndScoreNeeded() throws Exception {
                     assertThat(bucket.getDocCount(), equalTo(10L));
                     TopHits topHits = bucket.getAggregations().get("hits");
                     SearchHits hits = topHits.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(10L));
+                    assertThat(hits.getTotalHits().value(), equalTo(10L));
                     assertThat(hits.getHits().length, equalTo(3));
 
                     assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
@@ -501,7 +501,7 @@ public void testPagination() throws Exception {
                         assertThat(bucket.getDocCount(), equalTo(10L));
                         TopHits topHits = bucket.getAggregations().get("hits");
                         SearchHits hits = topHits.getHits();
-                        assertThat(hits.getTotalHits().value, equalTo(controlHits.getTotalHits().value));
+                        assertThat(hits.getTotalHits().value(), equalTo(controlHits.getTotalHits().value()));
                         assertThat(hits.getHits().length, equalTo(controlHits.getHits().length));
                         for (int i = 0; i < hits.getHits().length; i++) {
                             logger.info(
@@ -543,7 +543,7 @@ public void testSortByBucket() throws Exception {
                     assertThat(bucket.getDocCount(), equalTo(10L));
                     TopHits topHits = bucket.getAggregations().get("hits");
                     SearchHits hits = topHits.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(10L));
+                    assertThat(hits.getTotalHits().value(), equalTo(10L));
                     assertThat(hits.getHits().length, equalTo(3));
                     assertThat(hits.getAt(0).getSortValues()[0], equalTo(higestSortValue));
                     assertThat(hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
@@ -578,7 +578,7 @@ public void testFieldCollapsing() throws Exception {
                 assertThat(key(bucket), equalTo("b"));
                 TopHits topHits = bucket.getAggregations().get("hits");
                 SearchHits hits = topHits.getHits();
-                assertThat(hits.getTotalHits().value, equalTo(4L));
+                assertThat(hits.getTotalHits().value(), equalTo(4L));
                 assertThat(hits.getHits().length, equalTo(1));
                 assertThat(hits.getAt(0).getId(), equalTo("6"));
 
@@ -586,7 +586,7 @@ public void testFieldCollapsing() throws Exception {
                 assertThat(key(bucket), equalTo("c"));
                 topHits = bucket.getAggregations().get("hits");
                 hits = topHits.getHits();
-                assertThat(hits.getTotalHits().value, equalTo(3L));
+                assertThat(hits.getTotalHits().value(), equalTo(3L));
                 assertThat(hits.getHits().length, equalTo(1));
                 assertThat(hits.getAt(0).getId(), equalTo("9"));
 
@@ -594,7 +594,7 @@ public void testFieldCollapsing() throws Exception {
                 assertThat(key(bucket), equalTo("a"));
                 topHits = bucket.getAggregations().get("hits");
                 hits = topHits.getHits();
-                assertThat(hits.getTotalHits().value, equalTo(2L));
+                assertThat(hits.getTotalHits().value(), equalTo(2L));
                 assertThat(hits.getHits().length, equalTo(1));
                 assertThat(hits.getAt(0).getId(), equalTo("2"));
             }
@@ -630,7 +630,7 @@ public void testFetchFeatures() throws IOException {
                 for (Terms.Bucket bucket : terms.getBuckets()) {
                     TopHits topHits = bucket.getAggregations().get("hits");
                     SearchHits hits = topHits.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(10L));
+                    assertThat(hits.getTotalHits().value(), equalTo(10L));
                     assertThat(hits.getHits().length, equalTo(1));
 
                     SearchHit hit = hits.getAt(0);
@@ -682,7 +682,7 @@ public void testEmptyIndex() throws Exception {
             TopHits hits = response.getAggregations().get("hits");
             assertThat(hits, notNullValue());
             assertThat(hits.getName(), equalTo("hits"));
-            assertThat(hits.getHits().getTotalHits().value, equalTo(0L));
+            assertThat(hits.getHits().getTotalHits().value(), equalTo(0L));
         });
     }
 
@@ -744,7 +744,7 @@ public void testTopHitsInNestedSimple() throws Exception {
                 assertThat(bucket.getDocCount(), equalTo(1L));
                 TopHits topHits = bucket.getAggregations().get("top-comments");
                 SearchHits searchHits = topHits.getHits();
-                assertThat(searchHits.getTotalHits().value, equalTo(1L));
+                assertThat(searchHits.getTotalHits().value(), equalTo(1L));
                 assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
                 assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(1));
@@ -753,7 +753,7 @@ public void testTopHitsInNestedSimple() throws Exception {
                 assertThat(bucket.getDocCount(), equalTo(2L));
                 topHits = bucket.getAggregations().get("top-comments");
                 searchHits = topHits.getHits();
-                assertThat(searchHits.getTotalHits().value, equalTo(2L));
+                assertThat(searchHits.getTotalHits().value(), equalTo(2L));
                 assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
                 assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(2));
@@ -765,7 +765,7 @@ public void testTopHitsInNestedSimple() throws Exception {
                 assertThat(bucket.getDocCount(), equalTo(1L));
                 topHits = bucket.getAggregations().get("top-comments");
                 searchHits = topHits.getHits();
-                assertThat(searchHits.getTotalHits().value, equalTo(1L));
+                assertThat(searchHits.getTotalHits().value(), equalTo(1L));
                 assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
                 assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(4));
@@ -789,7 +789,7 @@ public void testTopHitsInSecondLayerNested() throws Exception {
                 assertThat(toComments.getDocCount(), equalTo(4L));
 
                 TopHits topComments = toComments.getAggregations().get("top-comments");
-                assertThat(topComments.getHits().getTotalHits().value, equalTo(4L));
+                assertThat(topComments.getHits().getTotalHits().value(), equalTo(4L));
                 assertThat(topComments.getHits().getHits().length, equalTo(4));
 
                 assertThat(topComments.getHits().getAt(0).getId(), equalTo("2"));
@@ -816,7 +816,7 @@ public void testTopHitsInSecondLayerNested() throws Exception {
                 assertThat(toReviewers.getDocCount(), equalTo(7L));
 
                 TopHits topReviewers = toReviewers.getAggregations().get("top-reviewers");
-                assertThat(topReviewers.getHits().getTotalHits().value, equalTo(7L));
+                assertThat(topReviewers.getHits().getTotalHits().value(), equalTo(7L));
                 assertThat(topReviewers.getHits().getHits().length, equalTo(7));
 
                 assertThat(topReviewers.getHits().getAt(0).getId(), equalTo("1"));
@@ -899,7 +899,7 @@ public void testNestedFetchFeatures() {
                 assertThat(nested.getDocCount(), equalTo(4L));
 
                 SearchHits hits = ((TopHits) nested.getAggregations().get("top-comments")).getHits();
-                assertThat(hits.getTotalHits().value, equalTo(4L));
+                assertThat(hits.getTotalHits().value(), equalTo(4L));
                 SearchHit searchHit = hits.getAt(0);
                 assertThat(searchHit.getId(), equalTo("1"));
                 assertThat(searchHit.getNestedIdentity().getField().string(), equalTo("comments"));
@@ -960,7 +960,7 @@ public void testTopHitsInNested() throws Exception {
 
                     TopHits hits = nested.getAggregations().get("comments");
                     SearchHits searchHits = hits.getHits();
-                    assertThat(searchHits.getTotalHits().value, equalTo(numNestedDocs));
+                    assertThat(searchHits.getTotalHits().value(), equalTo(numNestedDocs));
                     for (int j = 0; j < 3; j++) {
                         assertThat(searchHits.getAt(j).getNestedIdentity().getField().string(), equalTo("comments"));
                         assertThat(searchHits.getAt(j).getNestedIdentity().getOffset(), equalTo(0));
@@ -1064,7 +1064,7 @@ public void testNoStoredFields() throws Exception {
                     assertThat(bucket.getDocCount(), equalTo(10L));
                     TopHits topHits = bucket.getAggregations().get("hits");
                     SearchHits hits = topHits.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(10L));
+                    assertThat(hits.getTotalHits().value(), equalTo(10L));
                     assertThat(hits.getHits().length, equalTo(3));
                     for (SearchHit hit : hits) {
                         assertThat(hit.getSourceAsMap(), nullValue());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 3dee7a8d6e92f..6e00c1e5a8d90 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -67,7 +67,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     public void testUnmapped() throws Exception {
         assertResponse(prepareSearch("idx_unmapped").setQuery(matchAllQuery()).addAggregation(count("count").field("value")), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
 
             ValueCount valueCount = response.getAggregations().get("count");
             assertThat(valueCount, notNullValue());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
index 3263be081a6f7..2cd22c6a65222 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
@@ -72,14 +72,14 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas)
                     .setPreference(preference + Integer.toString(counter++))
                     .setQuery(QueryBuilders.termQuery("field", "test")),
                 searchResponse -> {
-                    if (searchResponse.getHits().getTotalHits().value != 1) {
+                    if (searchResponse.getHits().getTotalHits().value() != 1) {
                         refresh();
                         assertResponse(
                             client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")),
                             searchResponseAfterRefresh -> {
                                 logger.info(
                                     "hits count mismatch on any shard search failed, post explicit refresh hits are {}",
-                                    searchResponseAfterRefresh.getHits().getTotalHits().value
+                                    searchResponseAfterRefresh.getHits().getTotalHits().value()
                                 );
                                 ensureGreen();
                                 assertResponse(
@@ -88,7 +88,7 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas)
                                         .setQuery(QueryBuilders.termQuery("field", "test")),
                                     searchResponseAfterGreen -> logger.info(
                                         "hits count mismatch on any shard search failed, post explicit wait for green hits are {}",
-                                        searchResponseAfterGreen.getHits().getTotalHits().value
+                                        searchResponseAfterGreen.getHits().getTotalHits().value()
                                     )
                                 );
                             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
index cab70ba7d7339..0d06856ca1088 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
@@ -77,7 +77,7 @@ public void run() {
                         try {
                             while (stop.get() == false) {
                                 assertResponse(prepareSearch().setSize(numDocs), response -> {
-                                    if (response.getHits().getTotalHits().value != numDocs) {
+                                    if (response.getHits().getTotalHits().value() != numDocs) {
                                         // if we did not search all shards but had no serious failures that is potentially fine
                                         // if only the hit-count is wrong. this can happen if the cluster-state is behind when the
                                         // request comes in. It's a small window but a known limitation.
@@ -86,7 +86,7 @@ public void run() {
                                                 .allMatch(ssf -> ssf.getCause() instanceof NoShardAvailableActionException)) {
                                             nonCriticalExceptions.add(
                                                 "Count is "
-                                                    + response.getHits().getTotalHits().value
+                                                    + response.getHits().getTotalHits().value()
                                                     + " but "
                                                     + numDocs
                                                     + " was expected. "
@@ -100,7 +100,7 @@ public void run() {
                                     final SearchHits sh = response.getHits();
                                     assertThat(
                                         "Expected hits to be the same size the actual hits array",
-                                        sh.getTotalHits().value,
+                                        sh.getTotalHits().value(),
                                         equalTo((long) (sh.getHits().length))
                                     );
                                 });
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
index 1745ad82931ba..4b59d5b9a78d5 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
@@ -126,7 +126,7 @@ public void testDfsQueryThenFetch() throws Exception {
             .get();
         while (true) {
             assertNoFailures(searchResponse);
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             SearchHit[] hits = searchResponse.getHits().getHits();
             if (hits.length == 0) {
                 break; // finished
@@ -169,7 +169,7 @@ public void testDfsQueryThenFetchWithSort() throws Exception {
             .get();
         while (true) {
             assertNoFailures(searchResponse);
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             SearchHit[] hits = searchResponse.getHits().getHits();
             if (hits.length == 0) {
                 break; // finished
@@ -208,7 +208,7 @@ public void testQueryThenFetch() throws Exception {
             .get();
         while (true) {
             assertNoFailures(searchResponse);
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             SearchHit[] hits = searchResponse.getHits().getHits();
             if (hits.length == 0) {
                 break; // finished
@@ -237,7 +237,7 @@ public void testQueryThenFetchWithFrom() throws Exception {
         assertNoFailuresAndResponse(
             client().search(new SearchRequest("test").source(source.from(0).size(60)).searchType(QUERY_THEN_FETCH)),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
                 assertThat(searchResponse.getHits().getHits().length, equalTo(60));
                 for (int i = 0; i < 60; i++) {
                     SearchHit hit = searchResponse.getHits().getHits()[i];
@@ -248,7 +248,7 @@ public void testQueryThenFetchWithFrom() throws Exception {
         assertNoFailuresAndResponse(
             client().search(new SearchRequest("test").source(source.from(60).size(60)).searchType(QUERY_THEN_FETCH)),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
                 assertThat(searchResponse.getHits().getHits().length, equalTo(40));
                 for (int i = 0; i < 40; i++) {
                     SearchHit hit = searchResponse.getHits().getHits()[i];
@@ -271,7 +271,7 @@ public void testQueryThenFetchWithSort() throws Exception {
             .get();
         while (true) {
             assertNoFailures(searchResponse);
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             SearchHit[] hits = searchResponse.getHits().getHits();
             if (hits.length == 0) {
                 break; // finished
@@ -301,7 +301,7 @@ public void testSimpleFacets() throws Exception {
             .aggregation(AggregationBuilders.filter("test1", termQuery("name", "test1")));
 
         assertNoFailuresAndResponse(client().search(new SearchRequest("test").source(sourceBuilder)), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(100L));
 
             Global global = response.getAggregations().get("global");
             Filter all = global.getAggregations().get("all");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
index 223ee81e84a92..5984e1acc89af 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
@@ -685,7 +685,7 @@ public void testDateMathIndexes() throws ExecutionException, InterruptedExceptio
             assertNotNull(localClusterSearchInfo);
             Cluster remoteClusterSearchInfo = clusters.getCluster(REMOTE_CLUSTER);
             assertNotNull(remoteClusterSearchInfo);
-            assertThat(Objects.requireNonNull(response.getHits().getTotalHits()).value, greaterThan(2L));
+            assertThat(Objects.requireNonNull(response.getHits().getTotalHits()).value(), greaterThan(2L));
             for (var hit : response.getHits()) {
                 assertThat(hit.getIndex(), anyOf(equalTo("datemath-2001-01-01-14"), equalTo("remotemath-2001-01-01-14")));
             }
@@ -755,6 +755,70 @@ public void testNegativeRemoteIndexNameThrows() {
         assertNotNull(ee.getCause());
     }
 
+    public void testClusterDetailsWhenLocalClusterHasNoMatchingIndex() throws Exception {
+        Map<String, Object> testClusterInfo = setupTwoClusters();
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+        int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
+
+        SearchRequest searchRequest = new SearchRequest("nomatch*", REMOTE_CLUSTER + ":" + remoteIndex);
+        if (randomBoolean()) {
+            searchRequest = searchRequest.scroll(TimeValue.timeValueMinutes(1));
+        }
+
+        searchRequest.allowPartialSearchResults(false);
+        if (randomBoolean()) {
+            searchRequest.setBatchedReduceSize(randomIntBetween(3, 20));
+        }
+
+        boolean minimizeRoundtrips = false;
+        searchRequest.setCcsMinimizeRoundtrips(minimizeRoundtrips);
+
+        boolean dfs = randomBoolean();
+        if (dfs) {
+            searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
+        }
+
+        if (randomBoolean()) {
+            searchRequest.setPreFilterShardSize(1);
+        }
+
+        searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10));
+        assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> {
+            assertNotNull(response);
+
+            Clusters clusters = response.getClusters();
+            assertFalse("search cluster results should BE successful", clusters.hasPartialResults());
+            assertThat(clusters.getTotal(), equalTo(2));
+            assertThat(clusters.getClusterStateCount(Cluster.Status.SUCCESSFUL), equalTo(2));
+            assertThat(clusters.getClusterStateCount(Cluster.Status.SKIPPED), equalTo(0));
+            assertThat(clusters.getClusterStateCount(Cluster.Status.RUNNING), equalTo(0));
+            assertThat(clusters.getClusterStateCount(Cluster.Status.PARTIAL), equalTo(0));
+            assertThat(clusters.getClusterStateCount(Cluster.Status.FAILED), equalTo(0));
+
+            Cluster localClusterSearchInfo = clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+            assertNotNull(localClusterSearchInfo);
+            assertThat(localClusterSearchInfo.getStatus(), equalTo(Cluster.Status.SUCCESSFUL));
+            assertThat(localClusterSearchInfo.getIndexExpression(), equalTo("nomatch*"));
+            assertThat(localClusterSearchInfo.getTotalShards(), equalTo(0));
+            assertThat(localClusterSearchInfo.getSuccessfulShards(), equalTo(0));
+            assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(0));
+            assertThat(localClusterSearchInfo.getFailedShards(), equalTo(0));
+            assertThat(localClusterSearchInfo.getFailures().size(), equalTo(0));
+            assertThat(localClusterSearchInfo.getTook().millis(), equalTo(0L));
+
+            Cluster remoteClusterSearchInfo = clusters.getCluster(REMOTE_CLUSTER);
+            assertNotNull(remoteClusterSearchInfo);
+            assertThat(remoteClusterSearchInfo.getStatus(), equalTo(Cluster.Status.SUCCESSFUL));
+            assertThat(remoteClusterSearchInfo.getIndexExpression(), equalTo(remoteIndex));
+            assertThat(remoteClusterSearchInfo.getTotalShards(), equalTo(remoteNumShards));
+            assertThat(remoteClusterSearchInfo.getSuccessfulShards(), equalTo(remoteNumShards));
+            assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(0));
+            assertThat(remoteClusterSearchInfo.getFailedShards(), equalTo(0));
+            assertThat(remoteClusterSearchInfo.getFailures().size(), equalTo(0));
+            assertThat(remoteClusterSearchInfo.getTook().millis(), greaterThan(0L));
+        });
+    }
+
     private static void assertOneFailedShard(Cluster cluster, int totalShards) {
         assertNotNull(cluster);
         assertThat(cluster.getStatus(), equalTo(Cluster.Status.PARTIAL));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index 2cb2e186b257e..91cc344614c23 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -139,7 +139,7 @@ private void hitExecute(FetchContext context, HitContext hitContext) throws IOEx
                 hitField = new DocumentField(NAME, new ArrayList<>(1));
                 hitContext.hit().setDocumentField(NAME, hitField);
             }
-            Terms terms = hitContext.reader().getTermVector(hitContext.docId(), field);
+            Terms terms = hitContext.reader().termVectors().get(hitContext.docId(), field);
             if (terms != null) {
                 TermsEnum te = terms.iterator();
                 Map<String, Integer> tv = new HashMap<>();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
index 66d44a818b797..e39f8df9bad36 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
@@ -151,7 +151,7 @@ public void testSimpleNested() throws Exception {
                 assertSearchHit(response, 1, hasId("1"));
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-                assertThat(innerHits.getTotalHits().value, equalTo(2L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(2L));
                 assertThat(innerHits.getHits().length, equalTo(2));
                 assertThat(innerHits.getAt(0).getId(), equalTo("1"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
@@ -171,7 +171,7 @@ public void testSimpleNested() throws Exception {
                 assertThat(response.getHits().getAt(0).getShard(), notNullValue());
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
-                assertThat(innerHits.getTotalHits().value, equalTo(3L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(3L));
                 assertThat(innerHits.getHits().length, equalTo(3));
                 assertThat(innerHits.getAt(0).getId(), equalTo("2"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
@@ -196,7 +196,7 @@ public void testSimpleNested() throws Exception {
             ),
             response -> {
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
-                assertThat(innerHits.getTotalHits().value, equalTo(2L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(2L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 HighlightField highlightField = innerHits.getAt(0).getHighlightFields().get("comments.message");
                 assertThat(highlightField.fragments()[0].string(), equalTo("<em>fox</em> eat quick"));
@@ -264,7 +264,7 @@ public void testRandomNested() throws Exception {
                 SearchHit searchHit = response.getHits().getAt(i);
                 assertThat(searchHit.getShard(), notNullValue());
                 SearchHits inner = searchHit.getInnerHits().get("a");
-                assertThat(inner.getTotalHits().value, equalTo((long) field1InnerObjects[i]));
+                assertThat(inner.getTotalHits().value(), equalTo((long) field1InnerObjects[i]));
                 for (int j = 0; j < field1InnerObjects[i] && j < size; j++) {
                     SearchHit innerHit = inner.getAt(j);
                     assertThat(innerHit.getNestedIdentity().getField().string(), equalTo("field1"));
@@ -273,7 +273,7 @@ public void testRandomNested() throws Exception {
                 }
 
                 inner = searchHit.getInnerHits().get("b");
-                assertThat(inner.getTotalHits().value, equalTo((long) field2InnerObjects[i]));
+                assertThat(inner.getTotalHits().value(), equalTo((long) field2InnerObjects[i]));
                 for (int j = 0; j < field2InnerObjects[i] && j < size; j++) {
                     SearchHit innerHit = inner.getAt(j);
                     assertThat(innerHit.getNestedIdentity().getField().string(), equalTo("field2"));
@@ -378,13 +378,13 @@ public void testNestedMultipleLayers() throws Exception {
                 assertSearchHit(response, 1, hasId("1"));
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 assertThat(innerHits.getAt(0).getId(), equalTo("1"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
                 innerHits = innerHits.getAt(0).getInnerHits().get("remark");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 assertThat(innerHits.getAt(0).getId(), equalTo("1"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
@@ -409,13 +409,13 @@ public void testNestedMultipleLayers() throws Exception {
                 assertSearchHit(response, 1, hasId("1"));
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 assertThat(innerHits.getAt(0).getId(), equalTo("1"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
                 innerHits = innerHits.getAt(0).getInnerHits().get("remark");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 assertThat(innerHits.getAt(0).getId(), equalTo("1"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
@@ -436,7 +436,7 @@ public void testNestedMultipleLayers() throws Exception {
                 assertSearchHit(response, 1, hasId("2"));
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comments.remarks");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 assertThat(innerHits.getAt(0).getId(), equalTo("2"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
@@ -460,13 +460,13 @@ public void testNestedMultipleLayers() throws Exception {
                 assertSearchHit(response, 1, hasId("2"));
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 assertThat(innerHits.getAt(0).getId(), equalTo("2"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
                 innerHits = innerHits.getAt(0).getInnerHits().get("remark");
-                assertThat(innerHits.getTotalHits().value, equalTo(1L));
+                assertThat(innerHits.getTotalHits().value(), equalTo(1L));
                 assertThat(innerHits.getHits().length, equalTo(1));
                 assertThat(innerHits.getAt(0).getId(), equalTo("2"));
                 assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
@@ -538,7 +538,7 @@ public void testNestedDefinedAsObject() throws Exception {
             response -> {
                 assertHitCount(response, 1);
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
-                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getId(), equalTo("1"));
                 assertThat(
                     response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getNestedIdentity().getField().string(),
@@ -613,7 +613,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
                 SearchHit parent = response.getHits().getAt(0);
                 assertThat(parent.getId(), equalTo("1"));
                 SearchHits inner = parent.getInnerHits().get("comments.messages");
-                assertThat(inner.getTotalHits().value, equalTo(2L));
+                assertThat(inner.getTotalHits().value(), equalTo(2L));
                 assertThat(inner.getAt(0).getSourceAsString(), equalTo("{\"message\":\"no fox\"}"));
                 assertThat(inner.getAt(1).getSourceAsString(), equalTo("{\"message\":\"fox eat quick\"}"));
             }
@@ -629,7 +629,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
                 SearchHit hit = response.getHits().getAt(0);
                 assertThat(hit.getId(), equalTo("1"));
                 SearchHits messages = hit.getInnerHits().get("comments.messages");
-                assertThat(messages.getTotalHits().value, equalTo(2L));
+                assertThat(messages.getTotalHits().value(), equalTo(2L));
                 assertThat(messages.getAt(0).getId(), equalTo("1"));
                 assertThat(messages.getAt(0).getNestedIdentity().getField().string(), equalTo("comments.messages"));
                 assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(2));
@@ -651,7 +651,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
                 SearchHit hit = response.getHits().getAt(0);
                 assertThat(hit.getId(), equalTo("1"));
                 SearchHits messages = hit.getInnerHits().get("comments.messages");
-                assertThat(messages.getTotalHits().value, equalTo(1L));
+                assertThat(messages.getTotalHits().value(), equalTo(1L));
                 assertThat(messages.getAt(0).getId(), equalTo("1"));
                 assertThat(messages.getAt(0).getNestedIdentity().getField().string(), equalTo("comments.messages"));
                 assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
@@ -685,7 +685,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception {
                 SearchHit hit = response.getHits().getAt(0);
                 assertThat(hit.getId(), equalTo("1"));
                 SearchHits messages = hit.getInnerHits().get("comments.messages");
-                assertThat(messages.getTotalHits().value, equalTo(1L));
+                assertThat(messages.getTotalHits().value(), equalTo(1L));
                 assertThat(messages.getAt(0).getId(), equalTo("1"));
                 assertThat(messages.getAt(0).getNestedIdentity().getField().string(), equalTo("comments.messages"));
                 assertThat(messages.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
@@ -786,22 +786,22 @@ public void testMatchesQueriesNestedInnerHits() throws Exception {
         );
         assertNoFailuresAndResponse(prepareSearch("test").setQuery(query).setSize(numDocs).addSort("field1", SortOrder.ASC), response -> {
             assertAllSuccessful(response);
-            assertThat(response.getHits().getTotalHits().value, equalTo((long) numDocs));
+            assertThat(response.getHits().getTotalHits().value(), equalTo((long) numDocs));
             assertThat(response.getHits().getAt(0).getId(), equalTo("0"));
-            assertThat(response.getHits().getAt(0).getInnerHits().get("nested1").getTotalHits().value, equalTo(2L));
+            assertThat(response.getHits().getAt(0).getInnerHits().get("nested1").getTotalHits().value(), equalTo(2L));
             assertThat(response.getHits().getAt(0).getInnerHits().get("nested1").getAt(0).getMatchedQueries().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getInnerHits().get("nested1").getAt(0).getMatchedQueries()[0], equalTo("test1"));
             assertThat(response.getHits().getAt(0).getInnerHits().get("nested1").getAt(1).getMatchedQueries().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getInnerHits().get("nested1").getAt(1).getMatchedQueries()[0], equalTo("test3"));
 
             assertThat(response.getHits().getAt(1).getId(), equalTo("1"));
-            assertThat(response.getHits().getAt(1).getInnerHits().get("nested1").getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getAt(1).getInnerHits().get("nested1").getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getAt(1).getInnerHits().get("nested1").getAt(0).getMatchedQueries().length, equalTo(1));
             assertThat(response.getHits().getAt(1).getInnerHits().get("nested1").getAt(0).getMatchedQueries()[0], equalTo("test2"));
 
             for (int i = 2; i < numDocs; i++) {
                 assertThat(response.getHits().getAt(i).getId(), equalTo(String.valueOf(i)));
-                assertThat(response.getHits().getAt(i).getInnerHits().get("nested1").getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getAt(i).getInnerHits().get("nested1").getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(i).getInnerHits().get("nested1").getAt(0).getMatchedQueries().length, equalTo(1));
                 assertThat(response.getHits().getAt(i).getInnerHits().get("nested1").getAt(0).getMatchedQueries()[0], equalTo("test3"));
             }
@@ -844,7 +844,7 @@ public void testNestedSource() throws Exception {
             response -> {
                 assertHitCount(response, 1);
 
-                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(1));
                 assertThat(
                     response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"),
@@ -865,7 +865,7 @@ public void testNestedSource() throws Exception {
             response -> {
                 assertHitCount(response, 1);
 
-                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2));
                 assertThat(
                     response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"),
@@ -891,7 +891,7 @@ public void testNestedSource() throws Exception {
             ),
             response -> {
                 assertHitCount(response, 1);
-                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(0));
             }
         );
@@ -901,7 +901,7 @@ public void testNestedSource() throws Exception {
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None).innerHit(new InnerHitBuilder())),
             response -> {
                 assertHitCount(response, 1);
-                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits().value(), equalTo(2L));
                 assertFalse(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().isEmpty());
             }
         );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index 0ce4f34463b03..0805d0f366b0f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -3340,7 +3340,7 @@ public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOExcepti
                 new SearchSourceBuilder().query(query).highlighter(new HighlightBuilder().field("*").highlighterType(highlighterType))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getHighlightFields().get("text").fragments().length, equalTo(1));
             }
         );
@@ -3412,7 +3412,7 @@ public void testKeywordFieldHighlighting() throws IOException {
                     .highlighter(new HighlightBuilder().field("*"))
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 HighlightField highlightField = response.getHits().getAt(0).getHighlightFields().get("keyword_field");
                 assertThat(highlightField.fragments()[0].string(), equalTo("<em>some text</em>"));
             }
@@ -3569,7 +3569,7 @@ public void testHighlightQueryRewriteDatesWithNow() throws Exception {
                             .should(QueryBuilders.termQuery("field", "hello"))
                     ),
                 response -> {
-                    assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                     assertHighlight(response, 0, "field", 0, 1, equalTo("<x>hello</x> world"));
                 }
             );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index d1eb1ab533ab7..16e5e42e00c9f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -191,26 +191,26 @@ public void testStoredFields() throws Exception {
         indicesAdmin().prepareRefresh().get();
 
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("field1"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().get("field1").getValue().toString(), equalTo("value1"));
         });
         // field2 is not stored, check that it is not extracted from source.
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("field2"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(0));
             assertThat(response.getHits().getAt(0).getFields().get("field2"), nullValue());
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("field3"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().get("field3").getValue().toString(), equalTo("value3"));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("*3"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().get("field3").getValue().toString(), equalTo("value3"));
@@ -218,7 +218,7 @@ public void testStoredFields() throws Exception {
         assertResponse(
             prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").addStoredField("field1").addStoredField("field2"),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getFields().size(), equalTo(2));
                 assertThat(response.getHits().getAt(0).getFields().get("field3").getValue().toString(), equalTo("value3"));
@@ -226,20 +226,20 @@ public void testStoredFields() throws Exception {
             }
         );
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("field*"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(2));
             assertThat(response.getHits().getAt(0).getFields().get("field3").getValue().toString(), equalTo("value3"));
             assertThat(response.getHits().getAt(0).getFields().get("field1").getValue().toString(), equalTo("value1"));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("f*3"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getFields().get("field3").getValue().toString(), equalTo("value3"));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("*"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getSourceAsMap(), nullValue());
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(2));
@@ -247,7 +247,7 @@ public void testStoredFields() throws Exception {
             assertThat(response.getHits().getAt(0).getFields().get("field3").getValue().toString(), equalTo("value3"));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addStoredField("*").addStoredField("_source"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             assertThat(response.getHits().getAt(0).getSourceAsMap(), notNullValue());
             assertThat(response.getHits().getAt(0).getFields().size(), equalTo(2));
@@ -311,7 +311,7 @@ public void testScriptDocAndFields() throws Exception {
                     new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['date'].date.millis", Collections.emptyMap())
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertFalse(response.getHits().getAt(0).hasSource());
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 Set<String> fields = new HashSet<>(response.getHits().getAt(0).getFields().keySet());
@@ -342,7 +342,7 @@ public void testScriptDocAndFields() throws Exception {
                     new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value * factor", Map.of("factor", 2.0))
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 Set<String> fields = new HashSet<>(response.getHits().getAt(0).getFields().keySet());
                 assertThat(fields, equalTo(singleton("sNum1")));
@@ -429,7 +429,7 @@ public void testIdBasedScriptFields() throws Exception {
                 .setSize(numDocs)
                 .addScriptField("id", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_fields._id.value", Collections.emptyMap())),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo((long) numDocs));
+                assertThat(response.getHits().getTotalHits().value(), equalTo((long) numDocs));
                 for (int i = 0; i < numDocs; i++) {
                     assertThat(response.getHits().getAt(i).getId(), equalTo(Integer.toString(i)));
                     Set<String> fields = new HashSet<>(response.getHits().getAt(i).getFields().keySet());
@@ -638,7 +638,7 @@ public void testStoredFieldsWithoutSource() throws Exception {
                 .addStoredField("boolean_field")
                 .addStoredField("binary_field"),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 Set<String> fields = new HashSet<>(response.getHits().getAt(0).getFields().keySet());
                 assertThat(
@@ -681,7 +681,7 @@ public void testSearchFieldsMetadata() throws Exception {
             .get();
 
         assertResponse(prepareSearch("my-index").addStoredField("field1").addStoredField("_routing"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getAt(0).field("field1"), nullValue());
             assertThat(response.getHits().getAt(0).field("_routing").getValue().toString(), equalTo("1"));
         });
@@ -749,7 +749,7 @@ public void testGetFieldsComplexField() throws Exception {
         String field = "field1.field2.field3.field4";
 
         assertResponse(prepareSearch("my-index").addStoredField(field), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
             assertThat(response.getHits().getAt(0).field(field).getValues().get(0).toString(), equalTo("value1"));
             assertThat(response.getHits().getAt(0).field(field).getValues().get(1).toString(), equalTo("value2"));
@@ -866,7 +866,7 @@ public void testDocValueFields() throws Exception {
             builder.addDocValueField("*_field");
         }
         assertResponse(builder, response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             Set<String> fields = new HashSet<>(response.getHits().getAt(0).getFields().keySet());
             assertThat(
@@ -906,7 +906,7 @@ public void testDocValueFields() throws Exception {
             assertThat(response.getHits().getAt(0).getFields().get("ip_field").getValues(), equalTo(List.of("::1")));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).addDocValueField("*field"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             assertThat(response.getHits().getHits().length, equalTo(1));
             Set<String> fields = new HashSet<>(response.getHits().getAt(0).getFields().keySet());
             assertThat(
@@ -955,7 +955,7 @@ public void testDocValueFields() throws Exception {
                 .addDocValueField("double_field", "#.0")
                 .addDocValueField("date_field", "epoch_millis"),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 Set<String> fields = new HashSet<>(response.getHits().getAt(0).getFields().keySet());
                 assertThat(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
index 36e75435bb5de..76384253282de 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -250,7 +250,7 @@ public void testDistanceScoreGeoLinGaussExpWithOffset() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (numDummyDocs + 2)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (numDummyDocs + 2)));
                 assertThat(sh.getAt(0).getId(), anyOf(equalTo("1"), equalTo("2")));
                 assertThat(sh.getAt(1).getId(), anyOf(equalTo("1"), equalTo("2")));
                 assertThat(sh.getAt(1).getScore(), equalTo(sh.getAt(0).getScore()));
@@ -276,7 +276,7 @@ public void testDistanceScoreGeoLinGaussExpWithOffset() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (numDummyDocs + 2)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (numDummyDocs + 2)));
                 assertThat(sh.getAt(0).getId(), anyOf(equalTo("1"), equalTo("2")));
                 assertThat(sh.getAt(1).getId(), anyOf(equalTo("1"), equalTo("2")));
                 assertThat(sh.getAt(1).getScore(), equalTo(sh.getAt(0).getScore()));
@@ -300,7 +300,7 @@ public void testDistanceScoreGeoLinGaussExpWithOffset() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (numDummyDocs + 2)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (numDummyDocs + 2)));
                 assertThat(sh.getAt(0).getId(), anyOf(equalTo("1"), equalTo("2")));
                 assertThat(sh.getAt(1).getId(), anyOf(equalTo("1"), equalTo("2")));
                 assertThat(sh.getAt(1).getScore(), equalTo(sh.getAt(0).getScore()));
@@ -373,7 +373,7 @@ public void testBoostModeSettingWorks() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (2)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (2)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat(sh.getAt(1).getId(), equalTo("2"));
             }
@@ -386,7 +386,7 @@ public void testBoostModeSettingWorks() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (2)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (2)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat(sh.getAt(1).getId(), equalTo("2"));
             }
@@ -405,7 +405,7 @@ public void testBoostModeSettingWorks() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (2)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (2)));
                 assertThat(sh.getAt(0).getId(), equalTo("2"));
                 assertThat(sh.getAt(1).getId(), equalTo("1"));
             }
@@ -461,7 +461,7 @@ public void testParseGeoPoint() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo(1.0, 1.e-5));
             }
@@ -481,7 +481,7 @@ public void testParseGeoPoint() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo(1.0f, 1.e-5));
             }
@@ -528,7 +528,7 @@ public void testCombineModes() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo(1.0, 1.e-5));
             }
@@ -546,7 +546,7 @@ public void testCombineModes() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo(0.5, 1.e-5));
             }
@@ -564,7 +564,7 @@ public void testCombineModes() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo(2.0 + 0.5, 1.e-5));
                 logger.info(
@@ -588,7 +588,7 @@ public void testCombineModes() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo((2.0 + 0.5) / 2, 1.e-5));
             }
@@ -606,7 +606,7 @@ public void testCombineModes() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo(0.5, 1.e-5));
             }
@@ -624,7 +624,7 @@ public void testCombineModes() throws Exception {
             ),
             response -> {
                 SearchHits sh = response.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) (1)));
+                assertThat(sh.getTotalHits().value(), equalTo((long) (1)));
                 assertThat(sh.getAt(0).getId(), equalTo("1"));
                 assertThat((double) sh.getAt(0).getScore(), closeTo(2.0, 1.e-5));
             }
@@ -1131,7 +1131,7 @@ public void testMultiFieldOptions() throws Exception {
         assertResponse(client().search(new SearchRequest(new String[] {}).source(searchSource().query(baseQuery))), response -> {
             assertSearchHits(response, "1", "2");
             SearchHits sh = response.getHits();
-            assertThat(sh.getTotalHits().value, equalTo((long) (2)));
+            assertThat(sh.getTotalHits().value(), equalTo((long) (2)));
         });
 
         List<Float> lonlat = new ArrayList<>();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
index 7fb06c0b83015..a85d133450bec 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
@@ -144,7 +144,7 @@ public void testExplainScript() throws InterruptedException, IOException, Execut
             ),
             response -> {
                 SearchHits hits = response.getHits();
-                assertThat(hits.getTotalHits().value, equalTo(20L));
+                assertThat(hits.getTotalHits().value(), equalTo(20L));
                 int idCounter = 19;
                 for (SearchHit hit : hits.getHits()) {
                     assertThat(hit.getId(), equalTo(Integer.toString(idCounter)));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
index a0fe7e661020d..a38c9dc916056 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
@@ -145,9 +145,9 @@ public void testMinScoreFunctionScoreBasic() throws Exception {
             ),
             response -> {
                 if (score < minScore) {
-                    assertThat(response.getHits().getTotalHits().value, is(0L));
+                    assertThat(response.getHits().getTotalHits().value(), is(0L));
                 } else {
-                    assertThat(response.getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getHits().getTotalHits().value(), is(1L));
                 }
             }
         );
@@ -167,9 +167,9 @@ public void testMinScoreFunctionScoreBasic() throws Exception {
             ),
             response -> {
                 if (score < minScore) {
-                    assertThat(response.getHits().getTotalHits().value, is(0L));
+                    assertThat(response.getHits().getTotalHits().value(), is(0L));
                 } else {
-                    assertThat(response.getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getHits().getTotalHits().value(), is(1L));
                 }
             }
         );
@@ -224,9 +224,9 @@ public void testMinScoreFunctionScoreManyDocsAndRandomMinScore() throws IOExcept
 
     protected void assertMinScoreSearchResponses(int numDocs, SearchResponse searchResponse, int numMatchingDocs) {
         assertNoFailures(searchResponse);
-        assertThat((int) searchResponse.getHits().getTotalHits().value, is(numMatchingDocs));
+        assertThat((int) searchResponse.getHits().getTotalHits().value(), is(numMatchingDocs));
         int pos = 0;
-        for (int hitId = numDocs - 1; (numDocs - hitId) < searchResponse.getHits().getTotalHits().value; hitId--) {
+        for (int hitId = numDocs - 1; (numDocs - hitId) < searchResponse.getHits().getTotalHits().value(); hitId--) {
             assertThat(searchResponse.getHits().getAt(pos).getId(), equalTo(Integer.toString(hitId)));
             pos++;
         }
@@ -242,7 +242,7 @@ public void testWithEmptyFunctions() throws IOException, ExecutionException, Int
         assertNoFailuresAndResponse(
             client().search(new SearchRequest(new String[] {}).source(searchSource().explain(true).query(termQuery("text", "text")))),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 termQueryScore[0] = response.getHits().getAt(0).getScore();
             }
         );
@@ -259,7 +259,7 @@ protected void testMinScoreApplied(CombineFunction boostMode, float expectedScor
                 )
             ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getScore(), equalTo(expectedScore));
             }
         );
@@ -269,7 +269,7 @@ protected void testMinScoreApplied(CombineFunction boostMode, float expectedScor
                     searchSource().explain(true).query(functionScoreQuery(termQuery("text", "text")).boostMode(boostMode).setMinScore(2f))
                 )
             ),
-            response -> assertThat(response.getHits().getTotalHits().value, equalTo(0L))
+            response -> assertThat(response.getHits().getTotalHits().value(), equalTo(0L))
         );
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
index 025d224923dc0..9fed4ead8c248 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
@@ -9,19 +9,30 @@
 
 package org.elasticsearch.search.functionscore;
 
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.tests.util.English;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
+import org.elasticsearch.common.lucene.search.function.LeafScoreFunction;
+import org.elasticsearch.common.lucene.search.function.ScoreFunction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.collapse.CollapseBuilder;
@@ -29,11 +40,14 @@
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.List;
 
@@ -135,7 +149,7 @@ public void testRescorePhrase() throws Exception {
                     5
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getMaxScore(), equalTo(response.getHits().getHits()[0].getScore()));
                 assertThat(response.getHits().getHits()[0].getId(), equalTo("1"));
                 assertThat(response.getHits().getHits()[1].getId(), equalTo("3"));
@@ -415,7 +429,7 @@ private static void assertEquivalent(String query, SearchResponse plain, SearchR
         assertNoFailures(rescored);
         SearchHits leftHits = plain.getHits();
         SearchHits rightHits = rescored.getHits();
-        assertThat(leftHits.getTotalHits().value, equalTo(rightHits.getTotalHits().value));
+        assertThat(leftHits.getTotalHits().value(), equalTo(rightHits.getTotalHits().value()));
         assertThat(leftHits.getHits().length, equalTo(rightHits.getHits().length));
         SearchHit[] hits = leftHits.getHits();
         SearchHit[] rHits = rightHits.getHits();
@@ -841,7 +855,7 @@ public void testRescorePhaseWithInvalidSort() throws Exception {
                 .setTrackScores(true)
                 .addRescorer(new QueryRescorerBuilder(matchAllQuery()).setRescoreQueryWeight(100.0f), 50),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(5L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(5L));
                 assertThat(response.getHits().getHits().length, equalTo(5));
                 for (SearchHit hit : response.getHits().getHits()) {
                     assertThat(hit.getScore(), equalTo(101f));
@@ -888,7 +902,7 @@ public void testRescoreAfterCollapse() throws Exception {
             .addRescorer(new QueryRescorerBuilder(fieldValueScoreQuery("secondPassScore")))
             .setCollapse(new CollapseBuilder("group"));
         assertResponse(request, resp -> {
-            assertThat(resp.getHits().getTotalHits().value, equalTo(5L));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(5L));
             assertThat(resp.getHits().getHits().length, equalTo(3));
 
             SearchHit hit1 = resp.getHits().getAt(0);
@@ -968,7 +982,7 @@ public void testRescoreAfterCollapseRandom() throws Exception {
             .setSize(Math.min(numGroups, 10));
         long expectedNumHits = numHits;
         assertResponse(request, resp -> {
-            assertThat(resp.getHits().getTotalHits().value, equalTo(expectedNumHits));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(expectedNumHits));
             for (int pos = 0; pos < resp.getHits().getHits().length; pos++) {
                 SearchHit hit = resp.getHits().getAt(pos);
                 assertThat(hit.getId(), equalTo(sortedGroups[pos].id()));
@@ -979,9 +993,119 @@ public void testRescoreAfterCollapseRandom() throws Exception {
         });
     }
 
+    public void testRescoreWithTimeout() throws Exception {
+        // no dummy docs since merges can change scores while we run queries.
+        int numDocs = indexRandomNumbers("whitespace", -1, false);
+
+        String intToEnglish = English.intToEnglish(between(0, numDocs - 1));
+        String query = intToEnglish.split(" ")[0];
+        assertResponse(
+            prepareSearch().setSearchType(SearchType.QUERY_THEN_FETCH)
+                .setQuery(QueryBuilders.matchQuery("field1", query).operator(Operator.OR))
+                .setSize(10)
+                .addRescorer(new QueryRescorerBuilder(functionScoreQuery(new TestTimedScoreFunctionBuilder())).windowSize(100))
+                .setTimeout(TimeValue.timeValueMillis(10)),
+            r -> assertTrue(r.isTimedOut())
+        );
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestTimedQueryPlugin.class);
+    }
+
     private QueryBuilder fieldValueScoreQuery(String scoreField) {
         return functionScoreQuery(termQuery("shouldFilter", false), ScoreFunctionBuilders.fieldValueFactorFunction(scoreField)).boostMode(
             CombineFunction.REPLACE
         );
     }
+
+    public static class TestTimedQueryPlugin extends Plugin implements SearchPlugin {
+        @Override
+        public List<ScoreFunctionSpec<?>> getScoreFunctions() {
+            return List.of(
+                new ScoreFunctionSpec<>(
+                    new ParseField("timed"),
+                    TestTimedScoreFunctionBuilder::new,
+                    p -> new TestTimedScoreFunctionBuilder()
+                )
+            );
+        }
+    }
+
+    static class TestTimedScoreFunctionBuilder extends ScoreFunctionBuilder<TestTimedScoreFunctionBuilder> {
+        private final long time = 500;
+
+        TestTimedScoreFunctionBuilder() {}
+
+        TestTimedScoreFunctionBuilder(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        protected void doWriteTo(StreamOutput out) {}
+
+        @Override
+        public String getName() {
+            return "timed";
+        }
+
+        @Override
+        protected void doXContent(XContentBuilder builder, Params params) {}
+
+        @Override
+        protected boolean doEquals(TestTimedScoreFunctionBuilder functionBuilder) {
+            return false;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return 0;
+        }
+
+        @Override
+        protected ScoreFunction doToFunction(SearchExecutionContext context) throws IOException {
+            return new ScoreFunction(REPLACE) {
+                @Override
+                public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) throws IOException {
+                    return new LeafScoreFunction() {
+                        @Override
+                        public double score(int docId, float subQueryScore) {
+                            try {
+                                Thread.sleep(time);
+                            } catch (InterruptedException e) {
+                                Thread.currentThread().interrupt();
+                            }
+                            return time;
+                        }
+
+                        @Override
+                        public Explanation explainScore(int docId, Explanation subQueryScore) {
+                            return null;
+                        }
+                    };
+                }
+
+                @Override
+                public boolean needsScores() {
+                    return true;
+                }
+
+                @Override
+                protected boolean doEquals(ScoreFunction other) {
+                    return false;
+                }
+
+                @Override
+                protected int doHashCode() {
+                    return 0;
+                }
+            };
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.current();
+        }
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
index 7fdb31a468998..22e27d78531a6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
@@ -268,7 +268,7 @@ public void testSeedReportedInExplain() throws Exception {
                 .setExplain(true),
             response -> {
                 assertNoFailures(response);
-                assertEquals(1, response.getHits().getTotalHits().value);
+                assertEquals(1, response.getHits().getTotalHits().value());
                 SearchHit firstHit = response.getHits().getAt(0);
                 assertThat(firstHit.getExplanation().toString(), containsString("" + seed));
             }
@@ -283,12 +283,12 @@ public void testNoDocs() throws Exception {
             prepareSearch("test").setQuery(
                 functionScoreQuery(matchAllQuery(), randomFunction().seed(1234).setField(SeqNoFieldMapper.NAME))
             ),
-            response -> assertEquals(0, response.getHits().getTotalHits().value)
+            response -> assertEquals(0, response.getHits().getTotalHits().value())
         );
 
         assertNoFailuresAndResponse(
             prepareSearch("test").setQuery(functionScoreQuery(matchAllQuery(), randomFunction())),
-            response -> assertEquals(0, response.getHits().getTotalHits().value)
+            response -> assertEquals(0, response.getHits().getTotalHits().value())
         );
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index 9b574cb54a116..2fde645f0036b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -426,7 +426,7 @@ public void testExplain() throws Exception {
                 .setExplain(true),
             response -> {
                 assertNoFailures(response);
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 Explanation explanation = response.getHits().getHits()[0].getExplanation();
                 assertThat(explanation.getValue(), equalTo(response.getHits().getHits()[0].getScore()));
                 assertThat(explanation.toString(), startsWith("0.36464313 = Score based on 2 child docs in range from 0 to 1"));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
index 6993f24b895e0..e6cd89c09b979 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
@@ -147,10 +147,10 @@ public void testProfileMatchesRegular() throws Exception {
                 );
             }
 
-            if (vanillaResponse.getHits().getTotalHits().value != profileResponse.getHits().getTotalHits().value) {
+            if (vanillaResponse.getHits().getTotalHits().value() != profileResponse.getHits().getTotalHits().value()) {
                 Set<SearchHit> vanillaSet = new HashSet<>(Arrays.asList(vanillaResponse.getHits().getHits()));
                 Set<SearchHit> profileSet = new HashSet<>(Arrays.asList(profileResponse.getHits().getHits()));
-                if (vanillaResponse.getHits().getTotalHits().value > profileResponse.getHits().getTotalHits().value) {
+                if (vanillaResponse.getHits().getTotalHits().value() > profileResponse.getHits().getTotalHits().value()) {
                     vanillaSet.removeAll(profileSet);
                     fail("Vanilla hits were larger than profile hits.  Non-overlapping elements were: " + vanillaSet.toString());
                 } else {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java
index f263ececfdc7d..26b040e2309c2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java
@@ -133,7 +133,7 @@ public void testExists() throws Exception {
                                 response
                             ),
                             count,
-                            response.getHits().getTotalHits().value
+                            response.getHits().getTotalHits().value()
                         );
                     } catch (AssertionError e) {
                         for (SearchHit searchHit : allDocs.getHits()) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
index 96042e198ef43..0fd2bd6f94770 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
@@ -347,7 +347,7 @@ public void testPhraseType() {
                     ).type(MatchQueryParser.Type.PHRASE)
                 )
             ),
-            response -> assertThat(response.getHits().getTotalHits().value, greaterThan(1L))
+            response -> assertThat(response.getHits().getTotalHits().value(), greaterThan(1L))
         );
 
         assertSearchHitsWithoutFailures(
@@ -428,8 +428,8 @@ public void testSingleField() throws NoSuchFieldException, IllegalAccessExceptio
                         matchResp -> {
                             assertThat(
                                 "field: " + field + " query: " + builder.toString(),
-                                multiMatchResp.getHits().getTotalHits().value,
-                                equalTo(matchResp.getHits().getTotalHits().value)
+                                multiMatchResp.getHits().getTotalHits().value(),
+                                equalTo(matchResp.getHits().getTotalHits().value())
                             );
                             SearchHits hits = multiMatchResp.getHits();
                             if (field.startsWith("missing")) {
@@ -451,7 +451,7 @@ public void testEquivalence() {
         var response = prepareSearch("test").setSize(0).setQuery(matchAllQuery()).get();
         final int numDocs;
         try {
-            numDocs = (int) response.getHits().getTotalHits().value;
+            numDocs = (int) response.getHits().getTotalHits().value();
         } finally {
             response.decRef();
         }
@@ -944,7 +944,7 @@ private static void assertEquivalent(String query, SearchResponse left, SearchRe
         assertNoFailures(right);
         SearchHits leftHits = left.getHits();
         SearchHits rightHits = right.getHits();
-        assertThat(leftHits.getTotalHits().value, equalTo(rightHits.getTotalHits().value));
+        assertThat(leftHits.getTotalHits().value(), equalTo(rightHits.getTotalHits().value()));
         assertThat(leftHits.getHits().length, equalTo(rightHits.getHits().length));
         SearchHit[] hits = leftHits.getHits();
         SearchHit[] rHits = rightHits.getHits();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java
index e25e330e072a6..c8fe9498b156f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java
@@ -263,7 +263,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
     }
 
     private void assertHits(SearchHits hits, String... ids) {
-        assertThat(hits.getTotalHits().value, equalTo((long) ids.length));
+        assertThat(hits.getTotalHits().value(), equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
         for (SearchHit hit : hits.getHits()) {
             hitIds.add(hit.getId());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java
index 45b98686e0484..cffba49d5941c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -10,7 +10,7 @@
 package org.elasticsearch.search.query;
 
 import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.tests.analysis.MockTokenizer;
@@ -264,7 +264,7 @@ public void testConstantScoreQuery() throws Exception {
             MatchQueryBuilder matchQuery = matchQuery("f", English.intToEnglish(between(0, num)));
             final long[] constantScoreTotalHits = new long[1];
             assertResponse(prepareSearch("test_1").setQuery(constantScoreQuery(matchQuery)).setSize(num), response -> {
-                constantScoreTotalHits[0] = response.getHits().getTotalHits().value;
+                constantScoreTotalHits[0] = response.getHits().getTotalHits().value();
                 SearchHits hits = response.getHits();
                 for (SearchHit searchHit : hits) {
                     assertThat(searchHit, hasScore(1.0f));
@@ -277,7 +277,7 @@ public void testConstantScoreQuery() throws Exception {
                 ).setSize(num),
                 response -> {
                     SearchHits hits = response.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(constantScoreTotalHits[0]));
+                    assertThat(hits.getTotalHits().value(), equalTo(constantScoreTotalHits[0]));
                     if (constantScoreTotalHits[0] > 1) {
                         float expected = hits.getAt(0).getScore();
                         for (SearchHit searchHit : hits) {
@@ -1693,7 +1693,7 @@ public void testQueryStringParserCache() throws Exception {
         assertResponse(
             prepareSearch("test").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(QueryBuilders.queryStringQuery("xyz").boost(100)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 first[0] = response.getHits().getAt(0).getScore();
             }
@@ -1704,7 +1704,7 @@ public void testQueryStringParserCache() throws Exception {
                 prepareSearch("test").setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
                     .setQuery(QueryBuilders.queryStringQuery("xyz").boost(100)),
                 response -> {
-                    assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                     assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                     float actual = response.getHits().getAt(0).getScore();
                     assertThat(finalI + " expected: " + first[0] + " actual: " + actual, Float.compare(first[0], actual), equalTo(0));
@@ -1917,7 +1917,9 @@ public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
     }
 
     /**
-     * Test correct handling {@link SpanBooleanQueryRewriteWithMaxClause#rewrite(IndexReader, MultiTermQuery)}. That rewrite method is e.g.
+     * Test correct handling
+     * {@link SpanBooleanQueryRewriteWithMaxClause#rewrite(IndexSearcher, MultiTermQuery)}.
+     * That rewrite method is e.g.
      * set for fuzzy queries with "constant_score" rewrite nested inside a `span_multi` query and would cause NPEs due to an unset
      * {@link AttributeSource}.
      */
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index 35f11eb1429b4..522c20b687caa 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -609,7 +609,7 @@ public void testSimpleQueryStringWithAnalysisStopWords() throws Exception {
     }
 
     private void assertHits(SearchHits hits, String... ids) {
-        assertThat(hits.getTotalHits().value, equalTo((long) ids.length));
+        assertThat(hits.getTotalHits().value(), equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
         for (SearchHit hit : hits.getHits()) {
             hitIds.add(hit.getId());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
index 027b223e8c4ef..c8c1f50444c1d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
@@ -234,7 +234,7 @@ public String getWriteableName() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.RANK_FEATURE_PHASE_ADDED;
+            return TransportVersions.V_8_15_0;
         }
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java
index ce9ce7b9cf4cf..dbdd6e8c50027 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java
@@ -424,7 +424,7 @@ public String getWriteableName() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.RANK_FEATURE_PHASE_ADDED;
+            return TransportVersions.V_8_15_0;
         }
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java
index 13a7d1fa59496..97aa428822fae 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java
@@ -75,7 +75,7 @@ public void testSimpleSearch() throws ExecutionException, InterruptedException {
             assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.RUNNING), equalTo(0));
             assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.PARTIAL), equalTo(0));
             assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.FAILED), equalTo(0));
-            assertThat(response.getHits().getTotalHits().value, equalTo(testClusterInfo.get("total_docs")));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(testClusterInfo.get("total_docs")));
         });
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java
index 43197b77b2c1e..25b43a2dc946e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java
@@ -78,8 +78,8 @@ public void testRewrite() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(1L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_0"));
         });
     }
@@ -91,8 +91,8 @@ public void testRewriteCompound() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(1L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
         });
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java
index 35990fa3755b1..9a7ce2c5c28ab 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java
@@ -123,17 +123,17 @@ public void testSimplePreference() {
 
         assertResponse(
             prepareSearch().setQuery(matchAllQuery()),
-            response -> assertThat(response.getHits().getTotalHits().value, equalTo(1L))
+            response -> assertThat(response.getHits().getTotalHits().value(), equalTo(1L))
         );
 
         assertResponse(
             prepareSearch().setQuery(matchAllQuery()).setPreference("_local"),
-            response -> assertThat(response.getHits().getTotalHits().value, equalTo(1L))
+            response -> assertThat(response.getHits().getTotalHits().value(), equalTo(1L))
         );
 
         assertResponse(
             prepareSearch().setQuery(matchAllQuery()).setPreference("1234"),
-            response -> assertThat(response.getHits().getTotalHits().value, equalTo(1L))
+            response -> assertThat(response.getHits().getTotalHits().value(), equalTo(1L))
         );
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java
index 33b554a508e2b..06ce330213af8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java
@@ -51,15 +51,15 @@ public void testNodeSelection() {
         // Before we've gathered stats for all nodes, we should try each node once.
         Set<String> nodeIds = new HashSet<>();
         assertResponse(client.prepareSearch().setQuery(matchAllQuery()), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             nodeIds.add(response.getHits().getAt(0).getShard().getNodeId());
         });
         assertResponse(client.prepareSearch().setQuery(matchAllQuery()), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             nodeIds.add(response.getHits().getAt(0).getShard().getNodeId());
         });
         assertResponse(client.prepareSearch().setQuery(matchAllQuery()), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             nodeIds.add(response.getHits().getAt(0).getShard().getNodeId());
         });
         assertEquals(3, nodeIds.size());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java
index 2c96c27a0d12d..f59be6bb75928 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java
@@ -122,7 +122,7 @@ public void testCustomScriptBinaryField() throws Exception {
                     new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['binaryData'].get(0).length", emptyMap())
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
                 assertThat(response.getHits().getAt(0).getFields().get("sbinaryData").getValues().get(0), equalTo(16));
             }
@@ -175,7 +175,7 @@ public void testCustomScriptBoost() throws Exception {
                     new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
                 assertThat(response.getHits().getAt(0).getFields().get("sNum1").getValues().get(0), equalTo(2.0));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("3"));
@@ -196,7 +196,7 @@ public void testCustomScriptBoost() throws Exception {
                     new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("3"));
                 assertThat(response.getHits().getAt(0).getFields().get("sNum1").getValues().get(0), equalTo(3.0));
             }
@@ -214,7 +214,7 @@ public void testCustomScriptBoost() throws Exception {
                     new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(0).getFields().get("sNum1").getValues().get(0), equalTo(1.0));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index d3da4639a3927..ac5738a9b67b2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -44,7 +44,7 @@ public void testDuelQueryThenFetch() throws Exception {
             prepareSearch("index").setSearchType(context.searchType).addSort(context.sort).setSize(context.numDocs),
             control -> {
                 SearchHits sh = control.getHits();
-                assertThat(sh.getTotalHits().value, equalTo((long) context.numDocs));
+                assertThat(sh.getTotalHits().value(), equalTo((long) context.numDocs));
                 assertThat(sh.getHits().length, equalTo(context.numDocs));
 
                 SearchResponse searchScrollResponse = prepareSearch("index").setSearchType(context.searchType)
@@ -55,7 +55,7 @@ public void testDuelQueryThenFetch() throws Exception {
                 try {
 
                     assertNoFailures(searchScrollResponse);
-                    assertThat(searchScrollResponse.getHits().getTotalHits().value, equalTo((long) context.numDocs));
+                    assertThat(searchScrollResponse.getHits().getTotalHits().value(), equalTo((long) context.numDocs));
                     assertThat(searchScrollResponse.getHits().getHits().length, equalTo(context.scrollRequestSize));
 
                     int counter = 0;
@@ -69,7 +69,7 @@ public void testDuelQueryThenFetch() throws Exception {
                         searchScrollResponse.decRef();
                         searchScrollResponse = client().prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(10)).get();
                         assertNoFailures(searchScrollResponse);
-                        assertThat(searchScrollResponse.getHits().getTotalHits().value, equalTo((long) context.numDocs));
+                        assertThat(searchScrollResponse.getHits().getTotalHits().value(), equalTo((long) context.numDocs));
                         if (searchScrollResponse.getHits().getHits().length == 0) {
                             break;
                         }
@@ -241,7 +241,7 @@ private void testDuelIndexOrder(SearchType searchType, boolean trackScores, int
                 try {
                     while (true) {
                         assertNoFailures(scroll);
-                        assertEquals(control.getHits().getTotalHits().value, scroll.getHits().getTotalHits().value);
+                        assertEquals(control.getHits().getTotalHits().value(), scroll.getHits().getTotalHits().value());
                         assertEquals(control.getHits().getMaxScore(), scroll.getHits().getMaxScore(), 0.01f);
                         if (scroll.getHits().getHits().length == 0) {
                             break;
@@ -255,7 +255,7 @@ private void testDuelIndexOrder(SearchType searchType, boolean trackScores, int
                         scroll.decRef();
                         scroll = client().prepareSearchScroll(scroll.getScrollId()).setScroll(TimeValue.timeValueMinutes(10)).get();
                     }
-                    assertEquals(control.getHits().getTotalHits().value, scrollDocs);
+                    assertEquals(control.getHits().getTotalHits().value(), scrollDocs);
                 } catch (AssertionError e) {
                     logger.info("Control:\n{}", control);
                     logger.info("Scroll size={}, from={}:\n{}", size, scrollDocs, scroll);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index 7c3dde22ce9d0..7ac24b77a4b6d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -89,7 +89,7 @@ public void testSimpleScrollQueryThenFetch() throws Exception {
         try {
             long counter = 0;
 
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -98,7 +98,7 @@ public void testSimpleScrollQueryThenFetch() throws Exception {
             searchResponse.decRef();
             searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
 
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -107,7 +107,7 @@ public void testSimpleScrollQueryThenFetch() throws Exception {
             searchResponse.decRef();
             searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
 
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(30));
             for (SearchHit hit : searchResponse.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -145,7 +145,7 @@ public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws E
         try {
             long counter = 0;
 
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(3));
             for (SearchHit hit : searchResponse.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -155,7 +155,7 @@ public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws E
                 searchResponse.decRef();
                 searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
 
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
                 assertThat(searchResponse.getHits().getHits().length, equalTo(3));
                 for (SearchHit hit : searchResponse.getHits()) {
                     assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -166,7 +166,7 @@ public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws E
             searchResponse.decRef();
             searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
 
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(1));
             for (SearchHit hit : searchResponse.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -176,7 +176,7 @@ public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws E
             searchResponse.decRef();
             searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
 
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(0));
             for (SearchHit hit : searchResponse.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -262,7 +262,7 @@ public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception {
             .addSort("field", SortOrder.ASC)
             .get();
         try {
-            assertThat(searchResponse1.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse1.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse1.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse1.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter1++));
@@ -278,7 +278,7 @@ public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception {
             .addSort("field", SortOrder.ASC)
             .get();
         try {
-            assertThat(searchResponse2.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse2.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse2.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse2.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter2++));
@@ -289,7 +289,7 @@ public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception {
 
         searchResponse1 = client().prepareSearchScroll(searchResponse1.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
         try {
-            assertThat(searchResponse1.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse1.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse1.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse1.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter1++));
@@ -300,7 +300,7 @@ public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception {
 
         searchResponse2 = client().prepareSearchScroll(searchResponse2.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
         try {
-            assertThat(searchResponse2.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse2.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse2.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse2.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter2++));
@@ -381,7 +381,7 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception {
             .addSort("field", SortOrder.ASC)
             .get();
         try {
-            assertThat(searchResponse1.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse1.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse1.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse1.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter1++));
@@ -397,7 +397,7 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception {
             .addSort("field", SortOrder.ASC)
             .get();
         try {
-            assertThat(searchResponse2.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse2.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse2.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse2.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter2++));
@@ -408,7 +408,7 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception {
 
         searchResponse1 = client().prepareSearchScroll(searchResponse1.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
         try {
-            assertThat(searchResponse1.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse1.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse1.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse1.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter1++));
@@ -419,7 +419,7 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception {
 
         searchResponse2 = client().prepareSearchScroll(searchResponse2.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).get();
         try {
-            assertThat(searchResponse2.getHits().getTotalHits().value, equalTo(100L));
+            assertThat(searchResponse2.getHits().getTotalHits().value(), equalTo(100L));
             assertThat(searchResponse2.getHits().getHits().length, equalTo(35));
             for (SearchHit hit : searchResponse2.getHits()) {
                 assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter2++));
@@ -535,7 +535,7 @@ public void testCloseAndReopenOrDeleteWithActiveScroll() {
             prepareSearch().setQuery(matchAllQuery()).setSize(35).setScroll(TimeValue.timeValueMinutes(2)).addSort("field", SortOrder.ASC),
             searchResponse -> {
                 long counter = 0;
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(100L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(100L));
                 assertThat(searchResponse.getHits().getHits().length, equalTo(35));
                 for (SearchHit hit : searchResponse.getHits()) {
                     assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++));
@@ -601,7 +601,7 @@ public void testInvalidScrollKeepAlive() throws IOException {
 
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(1).setScroll(TimeValue.timeValueMinutes(5)), searchResponse -> {
             assertNotNull(searchResponse.getScrollId());
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(2L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(1));
             Exception ex = expectThrows(
                 Exception.class,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
index 7c459f91a1ac0..353858e9d6974 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
@@ -150,7 +150,7 @@ public void testWithNullStrings() throws InterruptedException {
                 .setQuery(matchAllQuery())
                 .searchAfter(new Object[] { 0, null }),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, Matchers.equalTo(2L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), Matchers.equalTo(2L));
                 assertThat(searchResponse.getHits().getHits().length, Matchers.equalTo(1));
                 assertThat(searchResponse.getHits().getHits()[0].getSourceAsMap().get("field1"), Matchers.equalTo(100));
                 assertThat(searchResponse.getHits().getHits()[0].getSourceAsMap().get("field2"), Matchers.equalTo("toto"));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index a62a042a3cab5..e87c4790aa665 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -555,7 +555,7 @@ public void testStrictlyCountRequest() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch("test_count_1", "test_count_2").setTrackTotalHits(true).setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(11L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(11L));
                 assertThat(response.getHits().getHits().length, equalTo(0));
             }
         );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java
index 979cb9e8a8c4c..e079994003751 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java
@@ -117,7 +117,7 @@ public void testWithPreferenceAndRoutings() throws Exception {
         setupIndex(totalDocs, numShards);
 
         assertResponse(prepareSearch("test").setQuery(matchAllQuery()).setPreference("_shards:1,4").setSize(0), sr -> {
-            int numDocs = (int) sr.getHits().getTotalHits().value;
+            int numDocs = (int) sr.getHits().getTotalHits().value();
             int max = randomIntBetween(2, numShards * 3);
             int fetchSize = randomIntBetween(10, 100);
             SearchRequestBuilder request = prepareSearch("test").setQuery(matchAllQuery())
@@ -129,7 +129,7 @@ public void testWithPreferenceAndRoutings() throws Exception {
         });
 
         assertResponse(prepareSearch("test").setQuery(matchAllQuery()).setRouting("foo", "bar").setSize(0), sr -> {
-            int numDocs = (int) sr.getHits().getTotalHits().value;
+            int numDocs = (int) sr.getHits().getTotalHits().value();
             int max = randomIntBetween(2, numShards * 3);
             int fetchSize = randomIntBetween(10, 100);
             SearchRequestBuilder request = prepareSearch("test").setQuery(matchAllQuery())
@@ -147,7 +147,7 @@ public void testWithPreferenceAndRoutings() throws Exception {
                 .addAliasAction(IndicesAliasesRequest.AliasActions.add().index("test").alias("alias3").routing("baz"))
         );
         assertResponse(prepareSearch("alias1", "alias3").setQuery(matchAllQuery()).setSize(0), sr -> {
-            int numDocs = (int) sr.getHits().getTotalHits().value;
+            int numDocs = (int) sr.getHits().getTotalHits().value();
             int max = randomIntBetween(2, numShards * 3);
             int fetchSize = randomIntBetween(10, 100);
             SearchRequestBuilder request = prepareSearch("alias1", "alias3").setQuery(matchAllQuery())
@@ -166,7 +166,7 @@ private void assertSearchSlicesWithScroll(SearchRequestBuilder request, String f
             SearchResponse searchResponse = request.slice(sliceBuilder).get();
             try {
                 totalResults += searchResponse.getHits().getHits().length;
-                int expectedSliceResults = (int) searchResponse.getHits().getTotalHits().value;
+                int expectedSliceResults = (int) searchResponse.getHits().getTotalHits().value();
                 int numSliceResults = searchResponse.getHits().getHits().length;
                 String scrollId = searchResponse.getScrollId();
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
@@ -238,7 +238,7 @@ private void assertSearchSlicesWithPointInTime(
 
             SearchResponse searchResponse = request.get();
             try {
-                int expectedSliceResults = (int) searchResponse.getHits().getTotalHits().value;
+                int expectedSliceResults = (int) searchResponse.getHits().getTotalHits().value();
 
                 while (true) {
                     int numHits = searchResponse.getHits().getHits().length;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
index 3be427e37d60c..d1841ebaf8071 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -286,7 +286,7 @@ public void testRandomSorting() throws IOException, InterruptedException, Execut
             assertNoFailuresAndResponse(
                 prepareSearch("test").setQuery(matchAllQuery()).setSize(size).addSort("dense_bytes", SortOrder.ASC),
                 response -> {
-                    assertThat(response.getHits().getTotalHits().value, equalTo((long) numDocs));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo((long) numDocs));
                     assertThat(response.getHits().getHits().length, equalTo(size));
                     Set<Entry<BytesRef, String>> entrySet = denseBytes.entrySet();
                     Iterator<Entry<BytesRef, String>> iterator = entrySet.iterator();
@@ -307,7 +307,7 @@ public void testRandomSorting() throws IOException, InterruptedException, Execut
                     .setSize(size)
                     .addSort("sparse_bytes", SortOrder.ASC),
                 response -> {
-                    assertThat(response.getHits().getTotalHits().value, equalTo((long) sparseBytes.size()));
+                    assertThat(response.getHits().getTotalHits().value(), equalTo((long) sparseBytes.size()));
                     assertThat(response.getHits().getHits().length, equalTo(size));
                     Set<Entry<BytesRef, String>> entrySet = sparseBytes.entrySet();
                     Iterator<Entry<BytesRef, String>> iterator = entrySet.iterator();
@@ -818,7 +818,7 @@ public void testSortMissingNumbers() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch().setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("3"));
                 assertThat(response.getHits().getAt(2).getId(), equalTo("2"));
@@ -828,7 +828,7 @@ public void testSortMissingNumbers() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch().setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("3"));
                 assertThat(response.getHits().getAt(2).getId(), equalTo("2"));
@@ -838,7 +838,7 @@ public void testSortMissingNumbers() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch().setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(2).getId(), equalTo("3"));
@@ -884,7 +884,7 @@ public void testSortMissingStrings() throws IOException {
             response -> {
                 assertThat(Arrays.toString(response.getShardFailures()), response.getFailedShards(), equalTo(0));
 
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("3"));
                 assertThat(response.getHits().getAt(2).getId(), equalTo("2"));
@@ -896,7 +896,7 @@ public void testSortMissingStrings() throws IOException {
             response -> {
                 assertThat(Arrays.toString(response.getShardFailures()), response.getFailedShards(), equalTo(0));
 
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("3"));
                 assertThat(response.getHits().getAt(2).getId(), equalTo("2"));
@@ -908,7 +908,7 @@ public void testSortMissingStrings() throws IOException {
             response -> {
                 assertThat(Arrays.toString(response.getShardFailures()), response.getFailedShards(), equalTo(0));
 
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(2).getId(), equalTo("3"));
@@ -920,7 +920,7 @@ public void testSortMissingStrings() throws IOException {
             response -> {
                 assertThat(Arrays.toString(response.getShardFailures()), response.getFailedShards(), equalTo(0));
 
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
                 assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
                 assertThat(response.getHits().getAt(2).getId(), equalTo("3"));
@@ -1183,7 +1183,7 @@ public void testSortMVField() throws Exception {
         refresh();
 
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("long_values", SortOrder.ASC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
@@ -1197,7 +1197,7 @@ public void testSortMVField() throws Exception {
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("long_values", SortOrder.DESC), response -> {
 
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1214,7 +1214,7 @@ public void testSortMVField() throws Exception {
                 .setSize(10)
                 .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode(SortMode.SUM)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getHits().length, equalTo(3));
 
                 assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1232,7 +1232,7 @@ public void testSortMVField() throws Exception {
                 .setSize(10)
                 .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode(SortMode.AVG)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getHits().length, equalTo(3));
 
                 assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1250,7 +1250,7 @@ public void testSortMVField() throws Exception {
                 .setSize(10)
                 .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode(SortMode.MEDIAN)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getHits().length, equalTo(3));
 
                 assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1264,7 +1264,7 @@ public void testSortMVField() throws Exception {
             }
         );
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("int_values", SortOrder.ASC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
@@ -1277,7 +1277,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).intValue(), equalTo(7));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("int_values", SortOrder.DESC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1290,7 +1290,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).intValue(), equalTo(3));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("short_values", SortOrder.ASC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
@@ -1303,7 +1303,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).intValue(), equalTo(7));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("short_values", SortOrder.DESC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1316,7 +1316,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).intValue(), equalTo(3));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("byte_values", SortOrder.ASC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
@@ -1329,7 +1329,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).intValue(), equalTo(7));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("byte_values", SortOrder.DESC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1342,7 +1342,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).intValue(), equalTo(3));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("float_values", SortOrder.ASC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
@@ -1355,7 +1355,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).floatValue(), equalTo(7f));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("float_values", SortOrder.DESC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1368,7 +1368,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).floatValue(), equalTo(3f));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("double_values", SortOrder.ASC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
@@ -1381,7 +1381,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).doubleValue(), equalTo(7d));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("double_values", SortOrder.DESC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1394,7 +1394,7 @@ public void testSortMVField() throws Exception {
             assertThat(((Number) response.getHits().getAt(2).getSortValues()[0]).doubleValue(), equalTo(3d));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("string_values", SortOrder.ASC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
@@ -1407,7 +1407,7 @@ public void testSortMVField() throws Exception {
             assertThat(response.getHits().getAt(2).getSortValues()[0], equalTo("07"));
         });
         assertResponse(prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("string_values", SortOrder.DESC), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
             assertThat(response.getHits().getHits().length, equalTo(3));
 
             assertThat(response.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
@@ -1719,8 +1719,8 @@ public void testSortDuelBetweenSingleShardAndMultiShardIndex() throws Exception
                 prepareSearch("test2").setFrom(from).setSize(size).addSort(sortField, order),
                 singleShardResponse -> {
                     assertThat(
-                        multiShardResponse.getHits().getTotalHits().value,
-                        equalTo(singleShardResponse.getHits().getTotalHits().value)
+                        multiShardResponse.getHits().getTotalHits().value(),
+                        equalTo(singleShardResponse.getHits().getTotalHits().value())
                     );
                     assertThat(multiShardResponse.getHits().getHits().length, equalTo(singleShardResponse.getHits().getHits().length));
                     for (int i = 0; i < multiShardResponse.getHits().getHits().length; i++) {
@@ -1747,14 +1747,14 @@ public void testCustomFormat() throws Exception {
         );
 
         assertNoFailuresAndResponse(prepareSearch("test").addSort(SortBuilders.fieldSort("ip")), response -> {
-            assertEquals(2, response.getHits().getTotalHits().value);
+            assertEquals(2, response.getHits().getTotalHits().value());
             assertArrayEquals(new String[] { "192.168.1.7" }, response.getHits().getAt(0).getSortValues());
             assertArrayEquals(new String[] { "2001:db8::ff00:42:8329" }, response.getHits().getAt(1).getSortValues());
         });
         assertNoFailuresAndResponse(
             prepareSearch("test").addSort(SortBuilders.fieldSort("ip")).searchAfter(new Object[] { "192.168.1.7" }),
             response -> {
-                assertEquals(2, response.getHits().getTotalHits().value);
+                assertEquals(2, response.getHits().getTotalHits().value());
                 assertEquals(1, response.getHits().getHits().length);
                 assertArrayEquals(new String[] { "2001:db8::ff00:42:8329" }, response.getHits().getAt(0).getSortValues());
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java
index ae0d2cbeb841f..fc5d40ae18c14 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java
@@ -362,7 +362,7 @@ public void testDocumentsWithNullValue() throws Exception {
         assertNoFailuresAndResponse(
             prepareSearch().setQuery(matchAllQuery()).addScriptField("id", scripField).addSort("svalue", SortOrder.ASC),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(searchResponse.getHits().getAt(0).field("id").getValue(), equalTo("1"));
                 assertThat(searchResponse.getHits().getAt(1).field("id").getValue(), equalTo("3"));
                 assertThat(searchResponse.getHits().getAt(2).field("id").getValue(), equalTo("2"));
@@ -373,7 +373,7 @@ public void testDocumentsWithNullValue() throws Exception {
                 .addScriptField("id", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['id'][0]", Collections.emptyMap()))
                 .addSort("svalue", SortOrder.ASC),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(searchResponse.getHits().getAt(0).field("id").getValue(), equalTo("1"));
                 assertThat(searchResponse.getHits().getAt(1).field("id").getValue(), equalTo("3"));
                 assertThat(searchResponse.getHits().getAt(2).field("id").getValue(), equalTo("2"));
@@ -391,7 +391,7 @@ public void testDocumentsWithNullValue() throws Exception {
                 }
                 assertThat(searchResponse.getFailedShards(), equalTo(0));
 
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(searchResponse.getHits().getAt(0).field("id").getValue(), equalTo("3"));
                 assertThat(searchResponse.getHits().getAt(1).field("id").getValue(), equalTo("1"));
                 assertThat(searchResponse.getHits().getAt(2).field("id").getValue(), equalTo("2"));
@@ -409,7 +409,7 @@ public void testDocumentsWithNullValue() throws Exception {
                 }
                 assertThat(searchResponse.getFailedShards(), equalTo(0));
 
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(searchResponse.getHits().getAt(0).field("id").getValue(), equalTo("2"));
             }
         );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java
index 6351d8d906389..ec9c680e17fc3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java
@@ -64,12 +64,12 @@ public void testInnerHits() {
                     )
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getAt(0).getId(), nullValue());
                 assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue());
                 assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
                 SearchHits hits = response.getHits().getAt(0).getInnerHits().get("nested");
-                assertThat(hits.getTotalHits().value, equalTo(1L));
+                assertThat(hits.getTotalHits().value(), equalTo(1L));
                 assertThat(hits.getAt(0).getId(), nullValue());
                 assertThat(hits.getAt(0).getSourceAsString(), nullValue());
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java
index 2952150c2cb22..f90056c6ae859 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java
@@ -54,10 +54,10 @@ public void testCustomBM25Similarity() throws Exception {
             .get();
 
         assertResponse(prepareSearch().setQuery(matchQuery("field1", "quick brown fox")), bm25SearchResponse -> {
-            assertThat(bm25SearchResponse.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(bm25SearchResponse.getHits().getTotalHits().value(), equalTo(1L));
             float bm25Score = bm25SearchResponse.getHits().getHits()[0].getScore();
             assertResponse(prepareSearch().setQuery(matchQuery("field2", "quick brown fox")), booleanSearchResponse -> {
-                assertThat(booleanSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(booleanSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 float defaultScore = booleanSearchResponse.getHits().getHits()[0].getScore();
                 assertThat(bm25Score, not(equalTo(defaultScore)));
             });
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
index 70b748c86ec96..89fc5f676cb1e 100644
--- a/server/src/main/java/module-info.java
+++ b/server/src/main/java/module-info.java
@@ -7,7 +7,6 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
 import org.elasticsearch.plugins.internal.RestExtension;
 
 /** The Elasticsearch Server Module. */
@@ -445,19 +444,22 @@
             org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.postings.ES812PostingsFormat;
-    provides org.apache.lucene.codecs.DocValuesFormat with ES87TSDBDocValuesFormat;
+    provides org.apache.lucene.codecs.DocValuesFormat with org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
     provides org.apache.lucene.codecs.KnnVectorsFormat
         with
             org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat,
             org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat,
             org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat,
             org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat,
-            org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat;
+            org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat,
+            org.elasticsearch.index.codec.vectors.ES816BinaryQuantizedVectorsFormat,
+            org.elasticsearch.index.codec.vectors.ES816HnswBinaryQuantizedVectorsFormat;
 
     provides org.apache.lucene.codecs.Codec
         with
             org.elasticsearch.index.codec.Elasticsearch814Codec,
-            org.elasticsearch.index.codec.Elasticsearch816Codec;
+            org.elasticsearch.index.codec.Elasticsearch816Codec,
+            org.elasticsearch.index.codec.Elasticsearch900Codec;
 
     provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider;
 
diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
index 5d04e31069b1c..32198ba7584be 100644
--- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -1819,12 +1819,6 @@ private enum ElasticsearchExceptionHandle {
             160,
             TransportVersions.V_7_10_0
         ),
-        VERSION_MISMATCH_EXCEPTION(
-            org.elasticsearch.action.search.VersionMismatchException.class,
-            org.elasticsearch.action.search.VersionMismatchException::new,
-            161,
-            TransportVersions.V_7_12_0
-        ),
         AUTHENTICATION_PROCESSING_ERROR(
             org.elasticsearch.ElasticsearchAuthenticationProcessingError.class,
             org.elasticsearch.ElasticsearchAuthenticationProcessingError::new,
@@ -1923,7 +1917,7 @@ private enum ElasticsearchExceptionHandle {
             ResourceAlreadyUploadedException.class,
             ResourceAlreadyUploadedException::new,
             181,
-            TransportVersions.ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION
+            TransportVersions.V_8_15_0
         ),
         INGEST_PIPELINE_EXCEPTION(
             org.elasticsearch.ingest.IngestPipelineException.class,
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 0f9c27a7877b8..25bb792d827a9 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -102,80 +102,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion V_8_13_0 = def(8_595_00_0);
     public static final TransportVersion V_8_13_4 = def(8_595_00_1);
     public static final TransportVersion V_8_14_0 = def(8_636_00_1);
-    // 8.15.0+
-    public static final TransportVersion WATERMARK_THRESHOLDS_STATS = def(8_637_00_0);
-    public static final TransportVersion ENRICH_CACHE_ADDITIONAL_STATS = def(8_638_00_0);
-    public static final TransportVersion ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED = def(8_639_00_0);
-    public static final TransportVersion ML_TRAINED_MODEL_CACHE_METADATA_ADDED = def(8_640_00_0);
-    public static final TransportVersion TOP_LEVEL_KNN_SUPPORT_QUERY_NAME = def(8_641_00_0);
-    public static final TransportVersion INDEX_SEGMENTS_VECTOR_FORMATS = def(8_642_00_0);
-    public static final TransportVersion ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION = def(8_643_00_0);
-    public static final TransportVersion ESQL_MV_ORDERING_SORTED_ASCENDING = def(8_644_00_0);
-    public static final TransportVersion ESQL_PAGE_MAPPING_TO_ITERATOR = def(8_645_00_0);
-    public static final TransportVersion BINARY_PIT_ID = def(8_646_00_0);
-    public static final TransportVersion SECURITY_ROLE_MAPPINGS_IN_CLUSTER_STATE = def(8_647_00_0);
-    public static final TransportVersion ESQL_REQUEST_TABLES = def(8_648_00_0);
-    public static final TransportVersion ROLE_REMOTE_CLUSTER_PRIVS = def(8_649_00_0);
-    public static final TransportVersion NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS = def(8_650_00_0);
-    public static final TransportVersion SHUTDOWN_REQUEST_TIMEOUTS_FIX = def(8_651_00_0);
-    public static final TransportVersion INDEXING_PRESSURE_REQUEST_REJECTIONS_COUNT = def(8_652_00_0);
-    public static final TransportVersion ROLLUP_USAGE = def(8_653_00_0);
-    public static final TransportVersion SECURITY_ROLE_DESCRIPTION = def(8_654_00_0);
-    public static final TransportVersion ML_INFERENCE_AZURE_OPENAI_COMPLETIONS = def(8_655_00_0);
-    public static final TransportVersion JOIN_STATUS_AGE_SERIALIZATION = def(8_656_00_0);
-    public static final TransportVersion ML_RERANK_DOC_OPTIONAL = def(8_657_00_0);
-    public static final TransportVersion FAILURE_STORE_FIELD_PARITY = def(8_658_00_0);
-    public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO = def(8_659_00_0);
-    public static final TransportVersion ML_INFERENCE_COHERE_COMPLETION_ADDED = def(8_660_00_0);
-    public static final TransportVersion ESQL_REMOVE_ES_SOURCE_OPTIONS = def(8_661_00_0);
-    public static final TransportVersion NODE_STATS_INGEST_BYTES = def(8_662_00_0);
-    public static final TransportVersion SEMANTIC_QUERY = def(8_663_00_0);
-    public static final TransportVersion GET_AUTOSCALING_CAPACITY_UNUSED_TIMEOUT = def(8_664_00_0);
-    public static final TransportVersion SIMULATE_VALIDATES_MAPPINGS = def(8_665_00_0);
-    public static final TransportVersion RULE_QUERY_RENAME = def(8_666_00_0);
-    public static final TransportVersion SPARSE_VECTOR_QUERY_ADDED = def(8_667_00_0);
-    public static final TransportVersion ESQL_ADD_INDEX_MODE_TO_SOURCE = def(8_668_00_0);
-    public static final TransportVersion GET_SHUTDOWN_STATUS_TIMEOUT = def(8_669_00_0);
-    public static final TransportVersion FAILURE_STORE_TELEMETRY = def(8_670_00_0);
-    public static final TransportVersion ADD_METADATA_FLATTENED_TO_ROLES = def(8_671_00_0);
-    public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED = def(8_672_00_0);
-    public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0);
-    public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0);
-    public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED = def(8_675_00_0);
-    public static final TransportVersion ADD_MISTRAL_EMBEDDINGS_INFERENCE = def(8_676_00_0);
-    public static final TransportVersion ML_CHUNK_INFERENCE_OPTION = def(8_677_00_0);
-    public static final TransportVersion RANK_FEATURE_PHASE_ADDED = def(8_678_00_0);
-    public static final TransportVersion RANK_DOC_IN_SHARD_FETCH_REQUEST = def(8_679_00_0);
-    public static final TransportVersion SECURITY_SETTINGS_REQUEST_TIMEOUTS = def(8_680_00_0);
-    public static final TransportVersion QUERY_RULE_CRUD_API_PUT = def(8_681_00_0);
-    public static final TransportVersion DROP_UNUSED_NODES_REQUESTS = def(8_682_00_0);
-    public static final TransportVersion QUERY_RULE_CRUD_API_GET_DELETE = def(8_683_00_0);
-    public static final TransportVersion MORE_LIGHTER_NODES_REQUESTS = def(8_684_00_0);
-    public static final TransportVersion DROP_UNUSED_NODES_IDS = def(8_685_00_0);
-    public static final TransportVersion DELETE_SNAPSHOTS_ASYNC_ADDED = def(8_686_00_0);
-    public static final TransportVersion VERSION_SUPPORTING_SPARSE_VECTOR_STATS = def(8_687_00_0);
-    public static final TransportVersion ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD = def(8_688_00_0);
-    public static final TransportVersion FAILURE_STORE_LAZY_CREATION = def(8_689_00_0);
-    public static final TransportVersion SNAPSHOT_REQUEST_TIMEOUTS = def(8_690_00_0);
-    public static final TransportVersion INDEX_METADATA_MAPPINGS_UPDATED_VERSION = def(8_691_00_0);
-    public static final TransportVersion ML_INFERENCE_ELAND_SETTINGS_ADDED = def(8_692_00_0);
-    public static final TransportVersion ML_ANTHROPIC_INTEGRATION_ADDED = def(8_693_00_0);
-    public static final TransportVersion ML_INFERENCE_GOOGLE_VERTEX_AI_EMBEDDINGS_ADDED = def(8_694_00_0);
-    public static final TransportVersion EVENT_INGESTED_RANGE_IN_CLUSTER_STATE = def(8_695_00_0);
-    public static final TransportVersion ESQL_ADD_AGGREGATE_TYPE = def(8_696_00_0);
-    public static final TransportVersion SECURITY_MIGRATIONS_MIGRATION_NEEDED_ADDED = def(8_697_00_0);
-    public static final TransportVersion K_FOR_KNN_QUERY_ADDED = def(8_698_00_0);
-    public static final TransportVersion TEXT_SIMILARITY_RERANKER_RETRIEVER = def(8_699_00_0);
-    public static final TransportVersion ML_INFERENCE_GOOGLE_VERTEX_AI_RERANKING_ADDED = def(8_700_00_0);
-    public static final TransportVersion VERSIONED_MASTER_NODE_REQUESTS = def(8_701_00_0);
-    public static final TransportVersion ML_INFERENCE_AMAZON_BEDROCK_ADDED = def(8_702_00_0);
-    public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15 = def(8_702_00_1);
-    public static final TransportVersion FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15 = def(8_702_00_2);
-    /**
-     * we made a single backport for ESQL_ES_FIELD_CACHED_SERIALIZATION and ESQL_ATTRIBUTE_CACHED_SERIALIZATION
-     * with only one TransportVersion entry
-     */
-    public static final TransportVersion ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15 = def(8_702_00_3);
+    public static final TransportVersion V_8_15_0 = def(8_702_00_2);
+    public static final TransportVersion V_8_15_2 = def(8_702_00_3);
     public static final TransportVersion ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS = def(8_703_00_0);
     public static final TransportVersion INFERENCE_ADAPTIVE_ALLOCATIONS = def(8_704_00_0);
     public static final TransportVersion INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN = def(8_705_00_0);
@@ -241,6 +169,19 @@ static TransportVersion def(int id) {
     public static final TransportVersion RETRIEVERS_TELEMETRY_ADDED = def(8_765_00_0);
     public static final TransportVersion ESQL_CACHED_STRING_SERIALIZATION = def(8_766_00_0);
     public static final TransportVersion CHUNK_SENTENCE_OVERLAP_SETTING_ADDED = def(8_767_00_0);
+    public static final TransportVersion OPT_IN_ESQL_CCS_EXECUTION_INFO = def(8_768_00_0);
+    public static final TransportVersion QUERY_RULE_TEST_API = def(8_769_00_0);
+    public static final TransportVersion ESQL_PER_AGGREGATE_FILTER = def(8_770_00_0);
+    public static final TransportVersion ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT = def(8_771_00_0);
+    public static final TransportVersion CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY = def(8_772_00_0);
+    public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16 = def(8_772_00_1);
+    public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0);
+    public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_00_0);
+    public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0);
+    public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ = def(8_776_00_0);
+    public static final TransportVersion SIMULATE_MAPPING_ADDITION = def(8_777_00_0);
+    public static final TransportVersion INTRODUCE_ALL_APPLICABLE_SELECTOR = def(8_778_00_0);
+    public static final TransportVersion INDEX_MODE_LOOKUP = def(8_779_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
@@ -307,7 +248,7 @@ static TransportVersion def(int id) {
      * Reference to the minimum transport version that can be used with CCS.
      * This should be the transport version used by the previous minor release.
      */
-    public static final TransportVersion MINIMUM_CCS_VERSION = FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15;
+    public static final TransportVersion MINIMUM_CCS_VERSION = V_8_15_0;
 
     static final NavigableMap<Integer, TransportVersion> VERSION_IDS = getAllVersionIds(TransportVersions.class);
 
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index 4b19d4b428526..5e4df05c10182 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -186,7 +186,9 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_15_1 = new Version(8_15_01_99);
     public static final Version V_8_15_2 = new Version(8_15_02_99);
     public static final Version V_8_15_3 = new Version(8_15_03_99);
+    public static final Version V_8_15_4 = new Version(8_15_04_99);
     public static final Version V_8_16_0 = new Version(8_16_00_99);
+    public static final Version V_8_17_0 = new Version(8_17_00_99);
     public static final Version V_9_0_0 = new Version(9_00_00_99);
     public static final Version CURRENT = V_9_0_0;
 
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 2d72f5d71ccda..08558b48c08b3 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -352,9 +352,7 @@
 import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestSimulateIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestSimulateTemplateAction;
-import org.elasticsearch.rest.action.admin.indices.RestSyncedFlushAction;
 import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction;
-import org.elasticsearch.rest.action.admin.indices.RestUpgradeActionDeprecated;
 import org.elasticsearch.rest.action.admin.indices.RestValidateQueryAction;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
 import org.elasticsearch.rest.action.cat.RestAliasAction;
@@ -916,7 +914,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
 
         registerHandler.accept(new RestRefreshAction());
         registerHandler.accept(new RestFlushAction());
-        registerHandler.accept(new RestSyncedFlushAction());
         registerHandler.accept(new RestForceMergeAction());
         registerHandler.accept(new RestClearIndicesCacheAction());
         registerHandler.accept(new RestResolveClusterAction());
@@ -1003,8 +1000,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestAnalyzeIndexDiskUsageAction());
         registerHandler.accept(new RestFieldUsageStatsAction());
 
-        registerHandler.accept(new RestUpgradeActionDeprecated());
-
         // Desired nodes
         registerHandler.accept(new RestGetDesiredNodesAction());
         registerHandler.accept(new RestUpdateDesiredNodesAction(clusterSupportsFeature));
diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index 095ccd71fa266..d47469ddf10d9 100644
--- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.seqno.SequenceNumbers;
@@ -298,9 +297,6 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
             builder.field(_SEQ_NO, getSeqNo());
             builder.field(_PRIMARY_TERM, getPrimaryTerm());
         }
-        if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-            builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-        }
         return builder;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
index 259a244bff919..e14f229f17acf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
@@ -156,7 +156,7 @@ public Response(Map<String, NodeAllocationStats> nodeAllocationStats, DiskThresh
         public Response(StreamInput in) throws IOException {
             super(in);
             this.nodeAllocationStats = in.readImmutableMap(StreamInput::readString, NodeAllocationStats::new);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.WATERMARK_THRESHOLDS_STATS)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 this.diskThresholdSettings = in.readOptionalWriteable(DiskThresholdSettings::readFrom);
             } else {
                 this.diskThresholdSettings = null;
@@ -166,7 +166,7 @@ public Response(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeMap(nodeAllocationStats, StreamOutput::writeString, StreamOutput::writeWriteable);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.WATERMARK_THRESHOLDS_STATS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 out.writeOptionalWriteable(diskThresholdSettings);
             } else {
                 assert diskThresholdSettings == null;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index a4fda469da3a2..83769430d4142 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -186,8 +186,7 @@ public static class NodeStatsRequest extends TransportRequest {
         public NodeStatsRequest(StreamInput in) throws IOException {
             super(in);
             this.nodesStatsRequestParameters = new NodesStatsRequestParameters(in);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)
-                && in.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_IDS)) {
+            if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_15_0)) {
                 in.readStringArray(); // formerly nodeIds, now unused
             }
         }
@@ -214,8 +213,7 @@ public String getDescription() {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             nodesStatsRequestParameters.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)
-                && out.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_IDS)) {
+            if (out.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_15_0)) {
                 out.writeStringArray(Strings.EMPTY_ARRAY); // formerly nodeIds, now unused
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java
index 8a321f484027f..9bdef4bfe4a13 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java
@@ -115,8 +115,7 @@ protected void doExecute(Task task, Request request, ActionListener<Response> li
         }
 
         private void executeWithSystemContext(Request request, ThreadContext threadContext, ActionListener<Response> listener) {
-            try (var ignore = threadContext.stashContext()) {
-                threadContext.markAsSystemContext();
+            try (var ignore = threadContext.newEmptySystemContext()) {
                 if (request.remoteClusterServer) {
                     final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().clear()
                         .addMetrics(NodesInfoMetrics.Metric.REMOTE_CLUSTER_SERVER.metricName());
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java
index 39fcb9fd53ac6..cdcf4bdad7b1a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java
@@ -29,7 +29,7 @@ public CleanupRepositoryRequest(TimeValue masterNodeTimeout, TimeValue ackTimeou
     }
 
     public static CleanupRepositoryRequest readFrom(StreamInput in) throws IOException {
-        if (in.getTransportVersion().onOrAfter(TransportVersions.SNAPSHOT_REQUEST_TIMEOUTS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             return new CleanupRepositoryRequest(in);
         } else {
             return new CleanupRepositoryRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, in);
@@ -48,7 +48,7 @@ public CleanupRepositoryRequest(TimeValue masterNodeTimeout, TimeValue ackTimeou
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.SNAPSHOT_REQUEST_TIMEOUTS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             super.writeTo(out);
         }
         out.writeString(repository);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
index 7b344a4c25a1b..45ee00a98c2e2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
@@ -92,17 +92,12 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
         if (emitState(outerParams)) {
             deprecationLogger.critical(DeprecationCategory.API, "reroute_cluster_state", STATE_FIELD_DEPRECATION_MESSAGE);
         }
-        return toXContentChunkedV7(outerParams);
-    }
-
-    @Override
-    public Iterator<? extends ToXContent> toXContentChunkedV7(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(b -> {
+        return ChunkedToXContent.builder(outerParams).object(b -> {
             b.field(ACKNOWLEDGED_KEY, isAcknowledged());
-            if (emitState(params)) {
+            if (emitState(outerParams)) {
                 b.xContentObject("state", state);
             }
-            if (params.paramAsBoolean("explain", false)) {
+            if (outerParams.paramAsBoolean("explain", false)) {
                 b.append(explanations);
             }
         });
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
index 7d3c31a011acc..ab073f83e14da 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
@@ -51,7 +51,7 @@ public DeleteSnapshotRequest(StreamInput in) throws IOException {
         super(in);
         repository = in.readString();
         snapshots = in.readStringArray();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.DELETE_SNAPSHOTS_ASYNC_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             waitForCompletion = in.readBoolean();
         }
     }
@@ -61,7 +61,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(repository);
         out.writeStringArray(snapshots);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.DELETE_SNAPSHOTS_ASYNC_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBoolean(waitForCompletion);
         } else {
             assert waitForCompletion : "Using wait_for_completion parameter when it should have been disallowed";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index b28c2a8e570ea..1dfd83aee0407 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -64,8 +64,7 @@ protected ClusterBlockException checkBlock(DeleteSnapshotRequest request, Cluste
 
     @Override
     protected void doExecute(Task task, DeleteSnapshotRequest request, ActionListener<AcknowledgedResponse> listener) {
-        if (clusterService.state().getMinTransportVersion().before(TransportVersions.DELETE_SNAPSHOTS_ASYNC_ADDED)
-            && request.waitForCompletion() == false) {
+        if (clusterService.state().getMinTransportVersion().before(TransportVersions.V_8_15_0) && request.waitForCompletion() == false) {
             throw new UnsupportedOperationException("wait_for_completion parameter is not supported by all nodes in this cluster");
         }
         super.doExecute(task, request, listener);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
index 080ebb5951a7a..553f784d23a87 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
@@ -35,6 +35,7 @@ public class CreateIndexClusterStateUpdateRequest {
     private ResizeType resizeType;
     private boolean copySettings;
     private SystemDataStreamDescriptor systemDataStreamDescriptor;
+    private boolean isFailureIndex = false;
 
     private Settings settings = Settings.EMPTY;
 
@@ -102,6 +103,11 @@ public CreateIndexClusterStateUpdateRequest systemDataStreamDescriptor(SystemDat
         return this;
     }
 
+    public CreateIndexClusterStateUpdateRequest isFailureIndex(boolean isFailureIndex) {
+        this.isFailureIndex = isFailureIndex;
+        return this;
+    }
+
     public String cause() {
         return cause;
     }
@@ -168,6 +174,10 @@ public String dataStreamName() {
         return dataStreamName;
     }
 
+    public boolean isFailureIndex() {
+        return isFailureIndex;
+    }
+
     public CreateIndexClusterStateUpdateRequest dataStreamName(String dataStreamName) {
         this.dataStreamName = dataStreamName;
         return this;
@@ -228,6 +238,8 @@ public String toString() {
             + systemDataStreamDescriptor
             + ", matchingTemplate="
             + matchingTemplate
+            + ", isFailureIndex="
+            + isFailureIndex
             + '}';
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
index f4ac1a3fe907b..c233ed57b748e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -113,7 +113,7 @@ public CreateIndexRequest(StreamInput in) throws IOException {
         } else {
             requireDataStream = false;
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_LAZY_CREATION)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             initializeFailureStore = in.readBoolean();
         } else {
             initializeFailureStore = true;
@@ -518,7 +518,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
             out.writeBoolean(this.requireDataStream);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_LAZY_CREATION)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBoolean(this.initializeFailureStore);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
index 6fe8432c31ccc..d942c4347960a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
@@ -79,7 +79,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
             builder.field("index_name", info.indexName);
             builder.field("index_uuid", info.indexUUID);
-            builder.timeField("creation_date_millis", "creation_date", info.creationDateMillis);
+            builder.timestampFieldsFromUnixEpochMillis("creation_date_millis", "creation_date", info.creationDateMillis);
 
             builder.array("node_ids", info.nodeIds.toArray(new String[0]));
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
index 666708ea6ffde..e668624440351 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
@@ -32,6 +32,7 @@
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.PointValues;
@@ -273,7 +274,7 @@ void analyzeDocValues(SegmentReader reader, IndexDiskUsageStats stats) throws IO
                 }
                 case SORTED_SET -> {
                     SortedSetDocValues sortedSet = iterateDocValues(maxDocs, () -> docValuesReader.getSortedSet(field), dv -> {
-                        while (dv.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
+                        for (int i = 0; i < dv.docValueCount(); i++) {
                             cancellationChecker.logEvent();
                         }
                     });
@@ -544,13 +545,14 @@ void analyzeKnnVectors(SegmentReader reader, IndexDiskUsageStats stats) throws I
             if (field.getVectorDimension() > 0) {
                 switch (field.getVectorEncoding()) {
                     case BYTE -> {
-                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getByteVectorValues(field.name), vectors -> {
+                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getByteVectorValues(field.name).iterator(), vectors -> {
                             cancellationChecker.logEvent();
-                            vectors.vectorValue();
+                            vectors.index();
                         });
 
                         // do a couple of randomized searches to figure out min and max offsets of index file
                         ByteVectorValues vectorValues = vectorReader.getByteVectorValues(field.name);
+                        KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
                         final KnnCollector collector = new TopKnnCollector(
                             Math.max(1, Math.min(100, vectorValues.size() - 1)),
                             Integer.MAX_VALUE
@@ -558,22 +560,23 @@ void analyzeKnnVectors(SegmentReader reader, IndexDiskUsageStats stats) throws I
                         int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
                         int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
                         for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
-                            if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
+                            if ((i = iterator.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
                                 break;
                             }
                             cancellationChecker.checkForCancellation();
-                            vectorReader.search(field.name, vectorValues.vectorValue(), collector, null);
+                            vectorReader.search(field.name, vectorValues.vectorValue(iterator.index()), collector, null);
                         }
                         stats.addKnnVectors(field.name, directory.getBytesRead());
                     }
                     case FLOAT32 -> {
-                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getFloatVectorValues(field.name), vectors -> {
+                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getFloatVectorValues(field.name).iterator(), vectors -> {
                             cancellationChecker.logEvent();
-                            vectors.vectorValue();
+                            vectors.index();
                         });
 
                         // do a couple of randomized searches to figure out min and max offsets of index file
                         FloatVectorValues vectorValues = vectorReader.getFloatVectorValues(field.name);
+                        KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
                         final KnnCollector collector = new TopKnnCollector(
                             Math.max(1, Math.min(100, vectorValues.size() - 1)),
                             Integer.MAX_VALUE
@@ -581,11 +584,11 @@ void analyzeKnnVectors(SegmentReader reader, IndexDiskUsageStats stats) throws I
                         int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
                         int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
                         for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
-                            if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
+                            if ((i = iterator.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
                                 break;
                             }
                             cancellationChecker.checkForCancellation();
-                            vectorReader.search(field.name, vectorValues.vectorValue(), collector, null);
+                            vectorReader.search(field.name, vectorValues.vectorValue(iterator.index()), collector, null);
                         }
                         stats.addKnnVectors(field.name, directory.getBytesRead());
                     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
index 7ff7066a15fc2..801dbbdee0858 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
@@ -98,9 +98,7 @@ public GetIndexRequest() {
         super(
             DataStream.isFailureStoreFeatureFlagEnabled()
                 ? IndicesOptions.builder(IndicesOptions.strictExpandOpen())
-                    .failureStoreOptions(
-                        IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)
-                    )
+                    .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                     .build()
                 : IndicesOptions.strictExpandOpen()
         );
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
index c3ed0c675c3c7..23a6b9c8c61a8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -30,9 +29,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY;
-import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
-
 /**
  * A response for a get index action.
  */
@@ -200,14 +196,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignore
                 if (indexMappings == null) {
                     builder.startObject("mappings").endObject();
                 } else {
-                    if (builder.getRestApiVersion() == RestApiVersion.V_7
-                        && params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) {
-                        builder.startObject("mappings");
-                        builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap());
-                        builder.endObject();
-                    } else {
-                        builder.field("mappings", indexMappings.sourceAsMap());
-                    }
+                    builder.field("mappings", indexMappings.sourceAsMap());
                 }
 
                 builder.startObject("settings");
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index 4398b33cd798f..01e8fe9787014 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -15,10 +15,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -31,8 +29,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY;
-
 /**
  * Response object for {@link GetFieldMappingsRequest} API
  *
@@ -91,16 +87,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.startObject(indexEntry.getKey());
             builder.startObject(MAPPINGS.getPreferredName());
             if (indexEntry.getValue() != null) {
-                if (builder.getRestApiVersion() == RestApiVersion.V_7
-                    && params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) {
-                    if (indexEntry.getValue().size() > 0) {
-                        builder.startObject(MapperService.SINGLE_MAPPING_NAME);
-                        addFieldMappingsToBuilder(builder, params, indexEntry.getValue());
-                        builder.endObject();
-                    }
-                } else {
-                    addFieldMappingsToBuilder(builder, params, indexEntry.getValue());
-                }
+                addFieldMappingsToBuilder(builder, params, indexEntry.getValue());
             }
 
             builder.endObject();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
index 37edae05c22af..cff2cf9ec8c78 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -26,9 +25,6 @@
 import java.util.Iterator;
 import java.util.Map;
 
-import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY;
-import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
-
 public class GetMappingsResponse extends ActionResponse implements ChunkedToXContentObject {
 
     private static final ParseField MAPPINGS = new ParseField("mappings");
@@ -73,16 +69,7 @@ public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
             Iterators.single((b, p) -> b.startObject()),
             Iterators.map(getMappings().entrySet().iterator(), indexEntry -> (builder, params) -> {
                 builder.startObject(indexEntry.getKey());
-                boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY);
-                if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName && indexEntry.getValue() != null) {
-                    builder.startObject(MAPPINGS.getPreferredName());
-
-                    if (indexEntry.getValue() != MappingMetadata.EMPTY_MAPPINGS) {
-                        builder.field(MapperService.SINGLE_MAPPING_NAME, indexEntry.getValue().sourceAsMap());
-                    }
-                    builder.endObject();
-
-                } else if (indexEntry.getValue() != null) {
+                if (indexEntry.getValue() != null) {
                     builder.field(MAPPINGS.getPreferredName(), indexEntry.getValue().sourceAsMap());
                 } else {
                     builder.startObject(MAPPINGS.getPreferredName()).endObject();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
index cb667400240f0..7857e9a22e9b9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.injection.guice.Inject;
@@ -119,18 +120,27 @@ public void onPrimaryOperationComplete(
             ActionListener<Void> listener
         ) {
             assert replicaRequest.primaryRefreshResult.refreshed() : "primary has not refreshed";
-            UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest(
-                indexShardRoutingTable,
-                replicaRequest.primaryRefreshResult.primaryTerm(),
-                replicaRequest.primaryRefreshResult.generation(),
-                false
-            );
-            transportService.sendRequest(
-                transportService.getLocalNode(),
-                TransportUnpromotableShardRefreshAction.NAME,
-                unpromotableReplicaRequest,
-                new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor)
+            boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(
+                clusterService.state().metadata().index(indexShardRoutingTable.shardId().getIndex()).getSettings()
             );
+
+            // Indices marked with fast refresh do not rely on refreshing the unpromotables
+            if (fastRefresh) {
+                listener.onResponse(null);
+            } else {
+                UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest(
+                    indexShardRoutingTable,
+                    replicaRequest.primaryRefreshResult.primaryTerm(),
+                    replicaRequest.primaryRefreshResult.generation(),
+                    false
+                );
+                transportService.sendRequest(
+                    transportService.getLocalNode(),
+                    TransportUnpromotableShardRefreshAction.NAME,
+                    unpromotableReplicaRequest,
+                    new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor)
+                );
+            }
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
index f91a983d47885..6c24ec2d17604 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
@@ -24,9 +24,6 @@
 
 import java.util.List;
 
-import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO;
-import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
-
 public class TransportUnpromotableShardRefreshAction extends TransportBroadcastUnpromotableAction<
     UnpromotableShardRefreshRequest,
     ActionResponse.Empty> {
@@ -76,18 +73,6 @@ protected void unpromotableShardOperation(
             return;
         }
 
-        // During an upgrade to FAST_REFRESH_RCO, we expect search shards to be first upgraded before the primary is upgraded. Thus,
-        // when the primary is upgraded, and starts to deliver unpromotable refreshes, we expect the search shards to be upgraded already.
-        // Note that the fast refresh setting is final.
-        // TODO: remove assertion (ES-9563)
-        assert INDEX_FAST_REFRESH_SETTING.get(shard.indexSettings().getSettings()) == false
-            || transportService.getLocalNodeConnection().getTransportVersion().onOrAfter(FAST_REFRESH_RCO)
-            : "attempted to refresh a fast refresh search shard "
-                + shard
-                + " on transport version "
-                + transportService.getLocalNodeConnection().getTransportVersion()
-                + " (before FAST_REFRESH_RCO)";
-
         ActionListener.run(responseListener, listener -> {
             shard.waitForPrimaryTermAndGeneration(
                 request.getPrimaryTerm(),
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index b6356e92ad856..552ce727d4249 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -13,12 +13,12 @@
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -42,7 +42,7 @@
  */
 public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
 
-    private static final ObjectParser<RolloverRequest, Boolean> PARSER = new ObjectParser<>("rollover");
+    private static final ObjectParser<RolloverRequest, Void> PARSER = new ObjectParser<>("rollover");
 
     private static final ParseField CONDITIONS = new ParseField("conditions");
 
@@ -57,28 +57,6 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
             CreateIndexRequest.SETTINGS,
             ObjectParser.ValueType.OBJECT
         );
-        PARSER.declareField((parser, request, includeTypeName) -> {
-            if (includeTypeName) {
-                // expecting one type only
-                for (Map.Entry<String, Object> mappingsEntry : parser.map().entrySet()) {
-                    @SuppressWarnings("unchecked")
-                    final Map<String, Object> value = (Map<String, Object>) mappingsEntry.getValue();
-                    request.createIndexRequest.mapping(value);
-                }
-            } else {
-                // a type is not included, add a dummy _doc type
-                Map<String, Object> mappings = parser.map();
-                if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) {
-                    throw new IllegalArgumentException(
-                        "The mapping definition cannot be nested under a type "
-                            + "["
-                            + MapperService.SINGLE_MAPPING_NAME
-                            + "] unless include_type_name is set to true."
-                    );
-                }
-                request.createIndexRequest.mapping(mappings);
-            }
-        }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT);
         PARSER.declareField((parser, request, context) -> {
             // a type is not included, add a dummy _doc type
             Map<String, Object> mappings = parser.map();
@@ -87,7 +65,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
                 throw new IllegalArgumentException("The mapping definition cannot be nested under a type");
             }
             request.createIndexRequest.mapping(mappings);
-        }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8)), ObjectParser.ValueType.OBJECT);
+        }, CreateIndexRequest.MAPPINGS, ObjectParser.ValueType.OBJECT);
 
         PARSER.declareField(
             (parser, request, context) -> request.createIndexRequest.aliases(parser.map()),
@@ -147,8 +125,8 @@ public ActionRequestValidationException validate() {
             );
         }
 
-        var failureStoreOptions = indicesOptions.failureStoreOptions();
-        if (failureStoreOptions.includeRegularIndices() && failureStoreOptions.includeFailureIndices()) {
+        var selector = indicesOptions.selectorOptions().defaultSelector();
+        if (selector == IndexComponentSelector.ALL_APPLICABLE) {
             validationException = addValidationError(
                 "rollover cannot be applied to both regular and failure indices at the same time",
                 validationException
@@ -188,7 +166,7 @@ public IndicesOptions indicesOptions() {
      * @return true of the rollover request targets the failure store, false otherwise.
      */
     public boolean targetsFailureStore() {
-        return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.failureStoreOptions().includeFailureIndices();
+        return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices();
     }
 
     public void setIndicesOptions(IndicesOptions indicesOptions) {
@@ -290,8 +268,8 @@ public CreateIndexRequest getCreateIndexRequest() {
     }
 
     // param isTypeIncluded decides how mappings should be parsed from XContent
-    public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException {
-        PARSER.parse(parser, this, isTypeIncluded);
+    public void fromXContent(XContentParser parser) throws IOException {
+        PARSER.parse(parser, this, null);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index d65a66dcc47fb..c5c874f9bcddf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -150,7 +150,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState
                 .matchClosed(request.indicesOptions().expandWildcardsClosed())
                 .build(),
             IndicesOptions.GatekeeperOptions.DEFAULT,
-            request.indicesOptions().failureStoreOptions()
+            request.indicesOptions().selectorOptions()
         );
 
         return state.blocks()
@@ -247,7 +247,7 @@ protected void masterOperation(
             IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS,
             IndicesOptions.WildcardOptions.builder().matchClosed(true).allowEmptyExpressions(false).build(),
             IndicesOptions.GatekeeperOptions.DEFAULT,
-            rolloverRequest.indicesOptions().failureStoreOptions()
+            rolloverRequest.indicesOptions().selectorOptions()
         );
         IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget())
             .clear()
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
index 1071f120f929e..45d784d301bf1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
@@ -16,10 +16,8 @@
 import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
@@ -120,9 +118,6 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                                                 builder.field(Fields.NUM_DOCS, segment.getNumDocs());
                                                 builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs());
                                                 builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize());
-                                                if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                                                    builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO);
-                                                }
                                                 builder.field(Fields.COMMITTED, segment.isCommitted());
                                                 builder.field(Fields.SEARCH, segment.isSearch());
                                                 if (segment.getVersion() != null) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java
index 26447910cecfc..bdacfb0ab642d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java
@@ -34,7 +34,7 @@ public IndicesSegmentsRequest(StreamInput in) throws IOException {
         if (in.getTransportVersion().before(TransportVersions.V_8_0_0)) {
             in.readBoolean();   // old 'verbose' option, since removed
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_SEGMENTS_VECTOR_FORMATS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.includeVectorFormatsInfo = in.readBoolean();
         }
     }
@@ -59,7 +59,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().before(TransportVersions.V_8_0_0)) {
             out.writeBoolean(false);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_SEGMENTS_VECTOR_FORMATS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBoolean(includeVectorFormatsInfo);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
index ddfe89cce01bf..1351674fffbfa 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
@@ -47,12 +47,11 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.TransportVersions.VERSION_SUPPORTING_SPARSE_VECTOR_STATS;
-
 public class CommonStats implements Writeable, ToXContentFragment {
 
     private static final TransportVersion VERSION_SUPPORTING_NODE_MAPPINGS = TransportVersions.V_8_5_0;
     private static final TransportVersion VERSION_SUPPORTING_DENSE_VECTOR_STATS = TransportVersions.V_8_10_X;
+    private static final TransportVersion VERSION_SUPPORTING_SPARSE_VECTOR_STATS = TransportVersions.V_8_15_0;
 
     @Nullable
     public DocsStats docs;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java
index 47abda4fabcde..347376a918d4c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java
@@ -69,7 +69,7 @@ public FieldUsageStats getStats() {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(Fields.TRACKING_ID, trackingId);
-        builder.timeField(Fields.TRACKING_STARTED_AT_MILLIS, Fields.TRACKING_STARTED_AT, trackingStartTime);
+        builder.timestampFieldsFromUnixEpochMillis(Fields.TRACKING_STARTED_AT_MILLIS, Fields.TRACKING_STARTED_AT, trackingStartTime);
         builder.startObject(Fields.ROUTING)
             .field(Fields.STATE, shardRouting.state())
             .field(Fields.PRIMARY, shardRouting.primary())
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
index fa3ac8ae720c2..2d854d2c6fa45 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -22,7 +21,6 @@
 import java.util.Objects;
 
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
 
 public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject {
 
@@ -65,11 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         builder.startObject();
         for (IndexTemplateMetadata indexTemplateMetadata : getIndexTemplates()) {
-            if (builder.getRestApiVersion() == RestApiVersion.V_7 && params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, false)) {
-                IndexTemplateMetadata.Builder.toXContentWithTypes(indexTemplateMetadata, builder, params);
-            } else {
-                IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params);
-            }
+            IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
index ec8eb4babfdac..94d9b87467ea8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
@@ -48,6 +48,7 @@
 
 import java.time.Instant;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -270,11 +271,12 @@ public static Template resolveTemplate(
         // First apply settings sourced from index settings providers
         final var now = Instant.now();
         Settings.Builder additionalSettings = Settings.builder();
+        Set<String> overrulingSettings = new HashSet<>();
         for (var provider : indexSettingProviders) {
             Settings result = provider.getAdditionalIndexSettings(
                 indexName,
                 template.getDataStreamTemplate() != null ? indexName : null,
-                template.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(template),
+                metadata.retrieveIndexModeFromTemplate(template),
                 simulatedState.getMetadata(),
                 now,
                 templateSettings,
@@ -283,8 +285,21 @@ public static Template resolveTemplate(
             MetadataCreateIndexService.validateAdditionalSettings(provider, result, additionalSettings);
             dummySettings.put(result);
             additionalSettings.put(result);
+            if (provider.overrulesTemplateAndRequestSettings()) {
+                overrulingSettings.addAll(result.keySet());
+            }
         }
-        // Then apply settings resolved from templates:
+
+        if (overrulingSettings.isEmpty() == false) {
+            // Filter any conflicting settings from overruling providers, to avoid overwriting their values from templates.
+            final Settings.Builder filtered = Settings.builder().put(templateSettings);
+            for (String setting : overrulingSettings) {
+                filtered.remove(setting);
+            }
+            templateSettings = filtered.build();
+        }
+
+        // Apply settings resolved from templates.
         dummySettings.put(templateSettings);
 
         final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java
index 78e603fba9be0..62a9b88cb6a57 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java
@@ -16,8 +16,10 @@
 
 import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS;
 import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS;
+import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_ADDITION;
 import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION;
 import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES;
+import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING;
 
 public class BulkFeatures implements FeatureSpecification {
     public Set<NodeFeature> getFeatures() {
@@ -25,7 +27,9 @@ public Set<NodeFeature> getFeatures() {
             SIMULATE_MAPPING_VALIDATION,
             SIMULATE_MAPPING_VALIDATION_TEMPLATES,
             SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS,
-            SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS
+            SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS,
+            SIMULATE_MAPPING_ADDITION,
+            SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
index c0ceab139ff1b..d5931c85bb2e1 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ShardId;
@@ -57,10 +56,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(STATUS, response.status().getStatus());
         } else {
             builder.field(_INDEX, failure.getIndex());
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-            }
-
             builder.field(_ID, failure.getId());
             builder.field(STATUS, failure.getStatus().getStatus());
             failure.getFailureStoreStatus().toXContent(builder, params);
@@ -301,9 +296,6 @@ public void setFailureStoreStatus(IndexDocFailureStoreStatus failureStoreStatus)
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(INDEX_FIELD, index);
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field("type", MapperService.SINGLE_MAPPING_NAME);
-            }
             if (id != null) {
                 builder.field(ID_FIELD, id);
             }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
index f04d07fb690c4..ce3e189149451 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
@@ -212,7 +212,7 @@ private void rollOverFailureStores(Runnable runnable) {
                 RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null);
                 rolloverRequest.setIndicesOptions(
                     IndicesOptions.builder(rolloverRequest.indicesOptions())
-                        .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true))
+                        .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
                         .build()
                 );
                 // We are executing a lazy rollover because it is an action specialised for this situation, when we want an
@@ -320,6 +320,12 @@ private Map<ShardId, List<BulkItemRequest>> groupRequestsByShards(
                     shard -> new ArrayList<>()
                 );
                 shardRequests.add(bulkItemRequest);
+            } catch (DataStream.TimestampError timestampError) {
+                IndexDocFailureStoreStatus failureStoreStatus = processFailure(bulkItemRequest, clusterState, timestampError);
+                if (IndexDocFailureStoreStatus.USED.equals(failureStoreStatus) == false) {
+                    String name = ia != null ? ia.getName() : docWriteRequest.index();
+                    addFailureAndDiscardRequest(docWriteRequest, bulkItemRequest.id(), name, timestampError, failureStoreStatus);
+                }
             } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException | ResourceNotFoundException e) {
                 String name = ia != null ? ia.getName() : docWriteRequest.index();
                 var failureStoreStatus = isFailureStoreRequest(docWriteRequest)
@@ -545,6 +551,7 @@ private IndexDocFailureStoreStatus processFailure(BulkItemRequest bulkItemReques
                 boolean added = addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreCandidate.getName());
                 if (added) {
                     failureStoreMetrics.incrementFailureStore(bulkItemRequest.index(), errorType, FailureStoreMetrics.ErrorLocation.SHARD);
+                    return IndexDocFailureStoreStatus.USED;
                 } else {
                     failureStoreMetrics.incrementRejected(
                         bulkItemRequest.index(),
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
index cd74989e5df7b..4c475bee985ab 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.seqno.SequenceNumbers;
-import org.elasticsearch.rest.action.document.RestBulkAction;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContent;
@@ -271,18 +270,11 @@ public int incrementalParse(
                                 }
                                 index = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
                             } else if (TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
-                                if (parser.getRestApiVersion().matches(RestApiVersion.equalTo(RestApiVersion.V_7))) {
-                                    // for bigger bulks, deprecation throttling might not be enough
-                                    if (deprecateOrErrorOnType && typesDeprecationLogged == false) {
-                                        deprecationLogger.compatibleCritical("bulk_with_types", RestBulkAction.TYPES_DEPRECATION_MESSAGE);
-                                        typesDeprecationLogged = true;
-                                    }
-                                } else if (parser.getRestApiVersion().matches(RestApiVersion.onOrAfter(RestApiVersion.V_8))
-                                    && deprecateOrErrorOnType) {
-                                        throw new IllegalArgumentException(
-                                            "Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]"
-                                        );
-                                    }
+                                if (deprecateOrErrorOnType) {
+                                    throw new IllegalArgumentException(
+                                        "Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]"
+                                    );
+                                }
                                 type = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
                             } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
                                 id = parser.text();
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
index 8d8f60cc3ba7b..406ba9723b427 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
@@ -43,7 +43,7 @@ public final class BulkShardRequest extends ReplicatedWriteRequest<BulkShardRequ
     public BulkShardRequest(StreamInput in) throws IOException {
         super(in);
         items = in.readArray(i -> i.readOptionalWriteable(inpt -> new BulkItemRequest(shardId, inpt)), BulkItemRequest[]::new);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             isSimulated = in.readBoolean();
         } else {
             isSimulated = false;
@@ -132,7 +132,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         super.writeTo(out);
         out.writeArray((o, item) -> o.writeOptional(BulkItemRequest.THIN_WRITER, item), items);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBoolean(isSimulated);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java
index f433e937dbe5d..a5a38a288d342 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java
@@ -18,12 +18,14 @@
 import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.common.xcontent.XContentElasticsearchExtension.DEFAULT_FORMATTER;
 import static org.elasticsearch.ingest.CompoundProcessor.PIPELINE_ORIGIN_EXCEPTION_HEADER;
 import static org.elasticsearch.ingest.CompoundProcessor.PROCESSOR_TAG_EXCEPTION_HEADER;
 import static org.elasticsearch.ingest.CompoundProcessor.PROCESSOR_TYPE_EXCEPTION_HEADER;
@@ -84,7 +86,7 @@ private static XContentBuilder createSource(IndexRequest source, Exception excep
         XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
         {
-            builder.timeField("@timestamp", timeSupplier.get());
+            builder.field("@timestamp", DEFAULT_FORMATTER.format(Instant.ofEpochMilli(timeSupplier.get())));
             builder.startObject("document");
             {
                 if (source.id() != null) {
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java
index 58ffe25e08e49..2e7c87301b2f6 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java
@@ -141,8 +141,7 @@ public void addItems(List<DocWriteRequest<?>> items, Releasable releasable, Runn
                     if (shouldBackOff()) {
                         final boolean isFirstRequest = incrementalRequestSubmitted == false;
                         incrementalRequestSubmitted = true;
-                        try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
-                            requestContext.restore();
+                        try (var ignored = threadContext.restoreExistingContext(requestContext)) {
                             final ArrayList<Releasable> toRelease = new ArrayList<>(releasables);
                             releasables.clear();
                             bulkInProgress = true;
@@ -188,8 +187,7 @@ public void lastItems(List<DocWriteRequest<?>> items, Releasable releasable, Act
             } else {
                 assert bulkRequest != null;
                 if (internalAddItems(items, releasable)) {
-                    try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
-                        requestContext.restore();
+                    try (var ignored = threadContext.restoreExistingContext(requestContext)) {
                         final ArrayList<Releasable> toRelease = new ArrayList<>(releasables);
                         releasables.clear();
                         // We do not need to set this back to false as this will be the last request.
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java
index 6fa22151396df..cc7fd431d8097 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java
@@ -15,12 +15,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * This extends BulkRequest with support for providing substitute pipeline definitions, component template definitions, and index template
@@ -73,7 +73,8 @@
  *           }
  *         }
  *       }
- *     },
+ *     }
+ *   },
  *   "index_template_substitutions": {
  *     "my-index-template-1": {
  *       "template": {
@@ -84,6 +85,13 @@
  *         ]
  *       }
  *     }
+ *   },
+ *   "mapping_addition": {
+ *     "dynamic": "strict",
+ *     "properties": {
+ *       "foo": {
+ *         "type": "keyword"
+ *       }
  *   }
  *
  *   The pipelineSubstitutions Map held by this class is intended to be the result of XContentHelper.convertToMap(). The top-level keys
@@ -94,6 +102,7 @@ public class SimulateBulkRequest extends BulkRequest {
     private final Map<String, Map<String, Object>> pipelineSubstitutions;
     private final Map<String, Map<String, Object>> componentTemplateSubstitutions;
     private final Map<String, Map<String, Object>> indexTemplateSubstitutions;
+    private final Map<String, Object> mappingAddition;
 
     /**
      * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with
@@ -103,16 +112,23 @@ public class SimulateBulkRequest extends BulkRequest {
      *                                       component template definitions with the same name.
      * @param indexTemplateSubstitutions The index template definitions that are to be used in place of any pre-existing
      *                                       index template definitions with the same name.
+     * @param mappingAddition   A mapping that will be merged into the final index's mapping for mapping validation
      */
     public SimulateBulkRequest(
-        @Nullable Map<String, Map<String, Object>> pipelineSubstitutions,
-        @Nullable Map<String, Map<String, Object>> componentTemplateSubstitutions,
-        @Nullable Map<String, Map<String, Object>> indexTemplateSubstitutions
+        Map<String, Map<String, Object>> pipelineSubstitutions,
+        Map<String, Map<String, Object>> componentTemplateSubstitutions,
+        Map<String, Map<String, Object>> indexTemplateSubstitutions,
+        Map<String, Object> mappingAddition
     ) {
         super();
+        Objects.requireNonNull(pipelineSubstitutions);
+        Objects.requireNonNull(componentTemplateSubstitutions);
+        Objects.requireNonNull(indexTemplateSubstitutions);
+        Objects.requireNonNull(mappingAddition);
         this.pipelineSubstitutions = pipelineSubstitutions;
         this.componentTemplateSubstitutions = componentTemplateSubstitutions;
         this.indexTemplateSubstitutions = indexTemplateSubstitutions;
+        this.mappingAddition = mappingAddition;
     }
 
     @SuppressWarnings("unchecked")
@@ -129,6 +145,11 @@ public SimulateBulkRequest(StreamInput in) throws IOException {
         } else {
             indexTemplateSubstitutions = Map.of();
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_MAPPING_ADDITION)) {
+            this.mappingAddition = (Map<String, Object>) in.readGenericValue();
+        } else {
+            mappingAddition = Map.of();
+        }
     }
 
     @Override
@@ -141,6 +162,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) {
             out.writeGenericValue(indexTemplateSubstitutions);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_MAPPING_ADDITION)) {
+            out.writeGenericValue(mappingAddition);
+        }
     }
 
     public Map<String, Map<String, Object>> getPipelineSubstitutions() {
@@ -153,41 +177,39 @@ public boolean isSimulated() {
     }
 
     @Override
-    public Map<String, ComponentTemplate> getComponentTemplateSubstitutions() throws IOException {
-        if (componentTemplateSubstitutions == null) {
-            return Map.of();
-        }
-        Map<String, ComponentTemplate> result = new HashMap<>(componentTemplateSubstitutions.size());
-        for (Map.Entry<String, Map<String, Object>> rawEntry : componentTemplateSubstitutions.entrySet()) {
-            result.put(rawEntry.getKey(), convertRawTemplateToComponentTemplate(rawEntry.getValue()));
-        }
-        return result;
+    public Map<String, ComponentTemplate> getComponentTemplateSubstitutions() {
+        return componentTemplateSubstitutions.entrySet()
+            .stream()
+            .collect(Collectors.toMap(Map.Entry::getKey, entry -> convertRawTemplateToComponentTemplate(entry.getValue())));
     }
 
     @Override
-    public Map<String, ComposableIndexTemplate> getIndexTemplateSubstitutions() throws IOException {
-        if (indexTemplateSubstitutions == null) {
-            return Map.of();
-        }
-        Map<String, ComposableIndexTemplate> result = new HashMap<>(indexTemplateSubstitutions.size());
-        for (Map.Entry<String, Map<String, Object>> rawEntry : indexTemplateSubstitutions.entrySet()) {
-            result.put(rawEntry.getKey(), convertRawTemplateToIndexTemplate(rawEntry.getValue()));
-        }
-        return result;
+    public Map<String, ComposableIndexTemplate> getIndexTemplateSubstitutions() {
+        return indexTemplateSubstitutions.entrySet()
+            .stream()
+            .collect(Collectors.toMap(Map.Entry::getKey, entry -> convertRawTemplateToIndexTemplate(entry.getValue())));
+    }
+
+    public Map<String, Object> getMappingAddition() {
+        return mappingAddition;
     }
 
-    private static ComponentTemplate convertRawTemplateToComponentTemplate(Map<String, Object> rawTemplate) throws IOException {
+    private static ComponentTemplate convertRawTemplateToComponentTemplate(Map<String, Object> rawTemplate) {
         ComponentTemplate componentTemplate;
         try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) {
             componentTemplate = ComponentTemplate.parse(parser);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
         }
         return componentTemplate;
     }
 
-    private static ComposableIndexTemplate convertRawTemplateToIndexTemplate(Map<String, Object> rawTemplate) throws IOException {
+    private static ComposableIndexTemplate convertRawTemplateToIndexTemplate(Map<String, Object> rawTemplate) {
         ComposableIndexTemplate indexTemplate;
         try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) {
             indexTemplate = ComposableIndexTemplate.parse(parser);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
         }
         return indexTemplate;
     }
@@ -197,7 +219,8 @@ public BulkRequest shallowClone() {
         BulkRequest bulkRequest = new SimulateBulkRequest(
             pipelineSubstitutions,
             componentTemplateSubstitutions,
-            indexTemplateSubstitutions
+            indexTemplateSubstitutions,
+            mappingAddition
         );
         bulkRequest.setRefreshPolicy(getRefreshPolicy());
         bulkRequest.waitForActiveShards(waitForActiveShards());
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 61adf41a9a276..cef68324e2a45 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -418,7 +418,7 @@ private void rollOverDataStreams(
             if (targetFailureStore) {
                 rolloverRequest.setIndicesOptions(
                     IndicesOptions.builder(rolloverRequest.indicesOptions())
-                        .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true))
+                        .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
                         .build()
                 );
             }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
index d7c555879c00f..1353fa78595ef 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
@@ -26,10 +26,13 @@
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
@@ -37,6 +40,7 @@
 import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexShard;
@@ -50,6 +54,10 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -75,6 +83,8 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction {
         "simulate.component.template.substitutions"
     );
     public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions");
+    public static final NodeFeature SIMULATE_MAPPING_ADDITION = new NodeFeature("simulate.mapping.addition");
+    public static final NodeFeature SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING = new NodeFeature("simulate.support.non.template.mapping");
     private final IndicesService indicesService;
     private final NamedXContentRegistry xContentRegistry;
     private final Set<IndexSettingProvider> indexSettingProviders;
@@ -122,11 +132,17 @@ protected void doInternalExecute(
         final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
         Map<String, ComponentTemplate> componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions();
         Map<String, ComposableIndexTemplate> indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions();
+        Map<String, Object> mappingAddition = ((SimulateBulkRequest) bulkRequest).getMappingAddition();
         for (int i = 0; i < bulkRequest.requests.size(); i++) {
             DocWriteRequest<?> docRequest = bulkRequest.requests.get(i);
             assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests";
             IndexRequest request = (IndexRequest) docRequest;
-            Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, indexTemplateSubstitutions, request);
+            Exception mappingValidationException = validateMappings(
+                componentTemplateSubstitutions,
+                indexTemplateSubstitutions,
+                mappingAddition,
+                request
+            );
             responses.set(
                 i,
                 BulkItemResponse.success(
@@ -159,6 +175,7 @@ protected void doInternalExecute(
     private Exception validateMappings(
         Map<String, ComponentTemplate> componentTemplateSubstitutions,
         Map<String, ComposableIndexTemplate> indexTemplateSubstitutions,
+        Map<String, Object> mappingAddition,
         IndexRequest request
     ) {
         final SourceToParse sourceToParse = new SourceToParse(
@@ -174,7 +191,10 @@ private Exception validateMappings(
         Exception mappingValidationException = null;
         IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index());
         try {
-            if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) {
+            if (indexAbstraction != null
+                && componentTemplateSubstitutions.isEmpty()
+                && indexTemplateSubstitutions.isEmpty()
+                && mappingAddition.isEmpty()) {
                 /*
                  * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService
                  * to do the mapping validation, using all the existing logic for validation.
@@ -239,6 +259,10 @@ private Exception validateMappings(
 
                 String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false);
                 if (matchingTemplate != null) {
+                    /*
+                     * The index matches a v2 template (including possibly one or more of the substitutions passed in). So we use this
+                     * template, and then possibly apply the mapping addition if it is not null, and validate.
+                     */
                     final Template template = TransportSimulateIndexTemplateAction.resolveTemplate(
                         matchingTemplate,
                         request.index(),
@@ -250,77 +274,40 @@ private Exception validateMappings(
                         indexSettingProviders
                     );
                     CompressedXContent mappings = template.mappings();
-                    if (mappings != null) {
-                        MappingMetadata mappingMetadata = new MappingMetadata(mappings);
-                        Settings dummySettings = Settings.builder()
-                            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-                            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-                            .build();
-                        final IndexMetadata imd = IndexMetadata.builder(request.index())
-                            .settings(dummySettings)
-                            .putMapping(mappingMetadata)
-                            .build();
-                        indicesService.withTempIndexService(imd, indexService -> {
-                            indexService.mapperService().updateMapping(null, imd);
-                            return IndexShard.prepareIndex(
-                                indexService.mapperService(),
-                                sourceToParse,
-                                SequenceNumbers.UNASSIGNED_SEQ_NO,
-                                -1,
-                                -1,
-                                VersionType.INTERNAL,
-                                Engine.Operation.Origin.PRIMARY,
-                                Long.MIN_VALUE,
-                                false,
-                                request.ifSeqNo(),
-                                request.ifPrimaryTerm(),
-                                0
-                            );
-                        });
-                    }
+                    CompressedXContent mergedMappings = mergeMappings(mappings, mappingAddition);
+                    validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse);
                 } else {
                     List<IndexTemplateMetadata> matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false);
-                    final Map<String, Object> mappingsMap = MetadataCreateIndexService.parseV1Mappings(
-                        "{}",
-                        matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
-                        xContentRegistry
-                    );
-                    final CompressedXContent combinedMappings;
-                    if (mappingsMap.isEmpty()) {
-                        combinedMappings = null;
+                    if (matchingTemplates.isEmpty() == false) {
+                        /*
+                         * The index matches v1 mappings. These are not compatible with component_template_substitutions or
+                         * index_template_substitutions, but we can apply a mapping_addition.
+                         */
+                        final Map<String, Object> mappingsMap = MetadataCreateIndexService.parseV1Mappings(
+                            "{}",
+                            matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
+                            xContentRegistry
+                        );
+                        final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
+                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                    } else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
+                        /*
+                         * The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
+                         * merge in the mapping addition if it exists, and validate.
+                         */
+                        MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping();
+                        CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
+                        CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
+                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
                     } else {
-                        combinedMappings = new CompressedXContent(mappingsMap);
+                        /*
+                         * The index matched no templates and had no mapping of its own. If there were component template substitutions
+                         * or index template substitutions, they didn't match anything. So just apply the mapping addition if it exists,
+                         * and validate.
+                         */
+                        final CompressedXContent combinedMappings = mergeMappings(null, mappingAddition);
+                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
                     }
-                    Settings dummySettings = Settings.builder()
-                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                        .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-                        .build();
-                    MappingMetadata mappingMetadata = combinedMappings == null ? null : new MappingMetadata(combinedMappings);
-                    final IndexMetadata imd = IndexMetadata.builder(request.index())
-                        .putMapping(mappingMetadata)
-                        .settings(dummySettings)
-                        .build();
-                    indicesService.withTempIndexService(imd, indexService -> {
-                        indexService.mapperService().updateMapping(null, imd);
-                        return IndexShard.prepareIndex(
-                            indexService.mapperService(),
-                            sourceToParse,
-                            SequenceNumbers.UNASSIGNED_SEQ_NO,
-                            -1,
-                            -1,
-                            VersionType.INTERNAL,
-                            Engine.Operation.Origin.PRIMARY,
-                            Long.MIN_VALUE,
-                            false,
-                            request.ifSeqNo(),
-                            request.ifPrimaryTerm(),
-                            0
-                        );
-                    });
                 }
             }
         } catch (Exception e) {
@@ -329,6 +316,66 @@ private Exception validateMappings(
         return mappingValidationException;
     }
 
+    /*
+     * Validates that when updatedMappings are applied
+     */
+    private void validateUpdatedMappings(
+        @Nullable CompressedXContent originalMappings,
+        @Nullable CompressedXContent updatedMappings,
+        IndexRequest request,
+        SourceToParse sourceToParse
+    ) throws IOException {
+        if (updatedMappings == null) {
+            return; // no validation to do
+        }
+        Settings dummySettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .build();
+        IndexMetadata.Builder originalIndexMetadataBuilder = IndexMetadata.builder(request.index()).settings(dummySettings);
+        if (originalMappings != null) {
+            originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings));
+        }
+        final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build();
+        final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index())
+            .settings(dummySettings)
+            .putMapping(new MappingMetadata(updatedMappings))
+            .build();
+        indicesService.withTempIndexService(originalIndexMetadata, indexService -> {
+            indexService.mapperService().merge(updatedIndexMetadata, MapperService.MergeReason.MAPPING_UPDATE);
+            return IndexShard.prepareIndex(
+                indexService.mapperService(),
+                sourceToParse,
+                SequenceNumbers.UNASSIGNED_SEQ_NO,
+                -1,
+                -1,
+                VersionType.INTERNAL,
+                Engine.Operation.Origin.PRIMARY,
+                Long.MIN_VALUE,
+                false,
+                request.ifSeqNo(),
+                request.ifPrimaryTerm(),
+                0
+            );
+        });
+    }
+
+    private static CompressedXContent mergeMappings(@Nullable CompressedXContent originalMapping, Map<String, Object> mappingAddition)
+        throws IOException {
+        Map<String, Object> combinedMappingMap = new HashMap<>();
+        if (originalMapping != null) {
+            combinedMappingMap.putAll(XContentHelper.convertToMap(originalMapping.uncompressed(), true, XContentType.JSON).v2());
+        }
+        XContentHelper.update(combinedMappingMap, mappingAddition, true);
+        if (combinedMappingMap.isEmpty()) {
+            return null;
+        } else {
+            return convertMappingMapToXContent(combinedMappingMap);
+        }
+    }
+
     /*
      * This overrides TransportSimulateBulkAction's getIngestService to allow us to provide an IngestService that handles pipeline
      * substitutions defined in the request.
@@ -344,4 +391,25 @@ protected Boolean resolveFailureStore(String indexName, Metadata metadata, long
         // A simulate bulk request should not change any persistent state in the system, so we never write to the failure store
         return null;
     }
+
+    private static CompressedXContent convertMappingMapToXContent(Map<String, Object> rawAdditionalMapping) throws IOException {
+        CompressedXContent compressedXContent;
+        if (rawAdditionalMapping == null || rawAdditionalMapping.isEmpty()) {
+            compressedXContent = null;
+        } else {
+            try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawAdditionalMapping)) {
+                compressedXContent = mappingFromXContent(parser);
+            }
+        }
+        return compressedXContent;
+    }
+
+    private static CompressedXContent mappingFromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        if (token == XContentParser.Token.START_OBJECT) {
+            return new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(parser.mapOrdered())));
+        } else {
+            throw new IllegalArgumentException("Unexpected token: " + token);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
index fbb084e8cd121..9266bae439b73 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
@@ -61,9 +61,7 @@ public Request() {
                             .allowFailureIndices(true)
                             .build()
                     )
-                    .failureStoreOptions(
-                        IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true).build()
-                    )
+                    .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                     .build()
             );
         }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java
index 2352628264394..f9bd135968246 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java
@@ -82,7 +82,7 @@ public ExplainIndexDataStreamLifecycle(
     public ExplainIndexDataStreamLifecycle(StreamInput in) throws IOException {
         this.index = in.readString();
         this.managedByLifecycle = in.readBoolean();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.isInternalDataStream = in.readBoolean();
         } else {
             this.isInternalDataStream = false;
@@ -123,7 +123,7 @@ public XContentBuilder toXContent(
         builder.field(MANAGED_BY_LIFECYCLE_FIELD.getPreferredName(), managedByLifecycle);
         if (managedByLifecycle) {
             if (indexCreationDate != null) {
-                builder.timeField(
+                builder.timestampFieldsFromUnixEpochMillis(
                     INDEX_CREATION_DATE_MILLIS_FIELD.getPreferredName(),
                     INDEX_CREATION_DATE_FIELD.getPreferredName(),
                     indexCreationDate
@@ -134,7 +134,11 @@ public XContentBuilder toXContent(
                 );
             }
             if (rolloverDate != null) {
-                builder.timeField(ROLLOVER_DATE_MILLIS_FIELD.getPreferredName(), ROLLOVER_DATE_FIELD.getPreferredName(), rolloverDate);
+                builder.timestampFieldsFromUnixEpochMillis(
+                    ROLLOVER_DATE_MILLIS_FIELD.getPreferredName(),
+                    ROLLOVER_DATE_FIELD.getPreferredName(),
+                    rolloverDate
+                );
                 builder.field(TIME_SINCE_ROLLOVER_FIELD.getPreferredName(), getTimeSinceRollover(nowSupplier).toHumanReadableString(2));
             }
             if (generationDateMillis != null) {
@@ -161,7 +165,7 @@ public XContentBuilder toXContent(
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
         out.writeBoolean(managedByLifecycle);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBoolean(isInternalDataStream);
         }
         if (managedByLifecycle) {
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
index bd628c88a1b1e..a43d29501a7ee 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
@@ -173,7 +173,7 @@ public record DataStreamLifecycle(
                 this(
                     in.readString(),
                     in.readOptionalWriteable(org.elasticsearch.cluster.metadata.DataStreamLifecycle::new),
-                    in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS) && in.readBoolean()
+                    in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) && in.readBoolean()
                 );
             }
 
@@ -181,7 +181,7 @@ public record DataStreamLifecycle(
             public void writeTo(StreamOutput out) throws IOException {
                 out.writeString(dataStreamName);
                 out.writeOptionalWriteable(lifecycle);
-                if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                     out.writeBoolean(isInternalDataStream);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
index caedec6a563d4..b759baad2024c 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.RestStatus;
@@ -138,10 +137,6 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(_INDEX.getPreferredName(), index);
-        if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-            builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-        }
-
         builder.field(_ID.getPreferredName(), id);
         builder.field(MATCHED.getPreferredName(), isMatch());
         if (hasExplanation()) {
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
index cc1d0497bd51a..22537f1f51216 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
@@ -26,11 +26,9 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceLoader;
-import org.elasticsearch.rest.action.document.RestMultiGetAction;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -454,47 +452,41 @@ private static void parseDocuments(
                         index = parser.text();
                     } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
                         id = parser.text();
-                    } else if (parser.getRestApiVersion() == RestApiVersion.V_7
-                        && TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
-                            deprecationLogger.compatibleCritical("mget_with_types", RestMultiGetAction.TYPES_DEPRECATION_MESSAGE);
-                        } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) {
-                            routing = parser.text();
-                        } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
-                            throw new ParsingException(
-                                parser.getTokenLocation(),
-                                "Unsupported field [fields] used, expected [stored_fields] instead"
-                            );
-                        } else if (STORED_FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
-                            storedFields = new ArrayList<>();
-                            storedFields.add(parser.text());
-                        } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
-                            version = parser.longValue();
-                        } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
-                            versionType = VersionType.fromString(parser.text());
-                        } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
-                            if (parser.isBooleanValue()) {
-                                fetchSourceContext = fetchSourceContext == null
-                                    ? FetchSourceContext.of(parser.booleanValue())
-                                    : FetchSourceContext.of(
-                                        parser.booleanValue(),
-                                        fetchSourceContext.includes(),
-                                        fetchSourceContext.excludes()
-                                    );
-                            } else if (token == Token.VALUE_STRING) {
-                                fetchSourceContext = FetchSourceContext.of(
-                                    fetchSourceContext == null || fetchSourceContext.fetchSource(),
-                                    new String[] { parser.text() },
-                                    fetchSourceContext == null ? Strings.EMPTY_ARRAY : fetchSourceContext.excludes()
+                    } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) {
+                        routing = parser.text();
+                    } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
+                        throw new ParsingException(
+                            parser.getTokenLocation(),
+                            "Unsupported field [fields] used, expected [stored_fields] instead"
+                        );
+                    } else if (STORED_FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
+                        storedFields = new ArrayList<>();
+                        storedFields.add(parser.text());
+                    } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
+                        version = parser.longValue();
+                    } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
+                        versionType = VersionType.fromString(parser.text());
+                    } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
+                        if (parser.isBooleanValue()) {
+                            fetchSourceContext = fetchSourceContext == null
+                                ? FetchSourceContext.of(parser.booleanValue())
+                                : FetchSourceContext.of(
+                                    parser.booleanValue(),
+                                    fetchSourceContext.includes(),
+                                    fetchSourceContext.excludes()
                                 );
-                            } else {
-                                throw new ElasticsearchParseException("illegal type for _source: [{}]", token);
-                            }
-                        } else {
-                            throw new ElasticsearchParseException(
-                                "failed to parse multi get request. unknown field [{}]",
-                                currentFieldName
+                        } else if (token == Token.VALUE_STRING) {
+                            fetchSourceContext = FetchSourceContext.of(
+                                fetchSourceContext == null || fetchSourceContext.fetchSource(),
+                                new String[] { parser.text() },
+                                fetchSourceContext == null ? Strings.EMPTY_ARRAY : fetchSourceContext.excludes()
                             );
+                        } else {
+                            throw new ElasticsearchParseException("illegal type for _source: [{}]", token);
                         }
+                    } else {
+                        throw new ElasticsearchParseException("failed to parse multi get request. unknown field [{}]", currentFieldName);
+                    }
                 } else if (token == Token.START_ARRAY) {
                     if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
                         throw new ParsingException(
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
index fc126b29b3265..08db6dee8e543 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -90,9 +89,6 @@ public void writeTo(StreamOutput out) throws IOException {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field(INDEX.getPreferredName(), index);
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-            }
             builder.field(ID.getPreferredName(), id);
             ElasticsearchException.generateFailureXContent(builder, params, exception, true);
             builder.endObject();
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index 99eac250641ae..9e535344c9589 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.admin.indices.refresh.TransportShardRefreshAction;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.action.support.replication.BasicReplicationRequest;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.client.internal.node.NodeClient;
@@ -125,10 +126,12 @@ protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionLi
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
         if (indexShard.routingEntry().isPromotableToPrimary() == false) {
+            // TODO: Re-evaluate assertion (ES-8227)
+            // assert indexShard.indexSettings().isFastRefresh() == false
+            // : "a search shard should not receive a TransportGetAction for an index with fast refresh";
             handleGetOnUnpromotableShard(request, indexShard, listener);
             return;
         }
-        // TODO: adapt assertion to assert only that it is not stateless (ES-9563)
         assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh()
             : "in Stateless a promotable to primary shard can receive a TransportGetAction only if an index has the fast refresh setting";
         if (request.realtime()) { // we are not tied to a refresh cycle here anyway
@@ -284,11 +287,11 @@ private void tryGetFromTranslog(GetRequest request, IndexShard indexShard, Disco
                     } else {
                         assert r.segmentGeneration() > -1L;
                         assert r.primaryTerm() > Engine.UNKNOWN_PRIMARY_TERM;
-                        indexShard.waitForPrimaryTermAndGeneration(
-                            r.primaryTerm(),
-                            r.segmentGeneration(),
-                            listener.delegateFailureAndWrap((ll, aLong) -> super.asyncShardOperation(request, shardId, ll))
+                        final ActionListener<Long> termAndGenerationListener = ContextPreservingActionListener.wrapPreservingContext(
+                            listener.delegateFailureAndWrap((ll, aLong) -> super.asyncShardOperation(request, shardId, ll)),
+                            threadPool.getThreadContext()
                         );
+                        indexShard.waitForPrimaryTermAndGeneration(r.primaryTerm(), r.segmentGeneration(), termAndGenerationListener);
                     }
                 }
             }), TransportGetFromTranslogAction.Response::new, getExecutor(request, shardId))
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index 633e7ef6793ab..34b3ae50e0b51 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -124,10 +124,12 @@ protected void asyncShardOperation(MultiGetShardRequest request, ShardId shardId
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
         if (indexShard.routingEntry().isPromotableToPrimary() == false) {
+            // TODO: Re-evaluate assertion (ES-8227)
+            // assert indexShard.indexSettings().isFastRefresh() == false
+            // : "a search shard should not receive a TransportShardMultiGetAction for an index with fast refresh";
             handleMultiGetOnUnpromotableShard(request, indexShard, listener);
             return;
         }
-        // TODO: adapt assertion to assert only that it is not stateless (ES-9563)
         assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh()
             : "in Stateless a promotable to primary shard can receive a TransportShardMultiGetAction only if an index has "
                 + "the fast refresh setting";
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java
index 1b930d5e8db3f..9d883cb075ede 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java
@@ -42,7 +42,7 @@ public SimulateIndexResponse(StreamInput in) throws IOException {
         this.source = in.readBytesReference();
         this.sourceXContentType = XContentType.valueOf(in.readString());
         setShardInfo(ShardInfo.EMPTY);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.exception = in.readException();
         } else {
             this.exception = null;
@@ -102,7 +102,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeBytesReference(source);
         out.writeString(sourceXContentType.name());
-        if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeException(exception);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
index 6570343452afe..12f542aec71c1 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.IngestDocument.Metadata;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -127,9 +125,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(key, value.toString());
             }
         }
-        if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-            builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-        }
         builder.field(SOURCE_FIELD, ingestDocument.getSource());
         builder.field(INGEST_FIELD, ingestDocument.getIngestMetadata());
         builder.endObject();
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index caa7453185575..0c585c705dcd0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.OriginalIndices;
@@ -234,15 +233,6 @@ public final void run() {
         }
         if (shardsIts.size() > 0) {
             doCheckNoMissingShards(getName(), request, shardsIts);
-            Version version = request.minCompatibleShardNode();
-            if (version != null && Version.CURRENT.minimumCompatibilityVersion().equals(version) == false) {
-                if (checkMinimumVersion(shardsIts) == false) {
-                    throw new VersionMismatchException(
-                        "One of the shards is incompatible with the required minimum version [{}]",
-                        request.minCompatibleShardNode()
-                    );
-                }
-            }
             for (int i = 0; i < shardsIts.size(); i++) {
                 final SearchShardIterator shardRoutings = shardsIts.get(i);
                 assert shardRoutings.skip() == false;
@@ -260,21 +250,6 @@ void skipShard(SearchShardIterator iterator) {
         successfulShardExecution(iterator);
     }
 
-    private boolean checkMinimumVersion(GroupShardsIterator<SearchShardIterator> shardsIts) {
-        for (SearchShardIterator it : shardsIts) {
-            if (it.getTargetNodeIds().isEmpty() == false) {
-                boolean isCompatible = it.getTargetNodeIds().stream().anyMatch(nodeId -> {
-                    Transport.Connection conn = getConnection(it.getClusterAlias(), nodeId);
-                    return conn == null || conn.getNode().getVersion().onOrAfter(request.minCompatibleShardNode());
-                });
-                if (isCompatible == false) {
-                    return false;
-                }
-            }
-        }
-        return true;
-    }
-
     private static boolean assertExecuteOnStartThread() {
         // Ensure that the current code has the following stacktrace:
         // AbstractSearchAsyncAction#start -> AbstractSearchAsyncAction#executePhase -> AbstractSearchAsyncAction#performPhaseOnShard
@@ -761,12 +736,7 @@ final void onPhaseDone() {  // as a tribute to @kimchy aka. finishHim()
 
     @Override
     public final Transport.Connection getConnection(String clusterAlias, String nodeId) {
-        Transport.Connection conn = nodeIdToConnection.apply(clusterAlias, nodeId);
-        Version minVersion = request.minCompatibleShardNode();
-        if (minVersion != null && conn != null && conn.getNode().getVersion().before(minVersion)) {
-            throw new VersionMismatchException("One of the shards is incompatible with the required minimum version [{}]", minVersion);
-        }
-        return conn;
+        return nodeIdToConnection.apply(clusterAlias, nodeId);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java
index 8ac2033e2ff19..dda589a458f88 100644
--- a/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java
+++ b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java
@@ -54,7 +54,7 @@ SearchSortValuesAndFormats getBottomSortValues() {
     }
 
     synchronized void consumeTopDocs(TopFieldDocs topDocs, DocValueFormat[] sortValuesFormat) {
-        totalHits += topDocs.totalHits.value;
+        totalHits += topDocs.totalHits.value();
         if (validateShardSortFields(topDocs.fields) == false) {
             return;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
index 8ce2cc7b6b19e..8dcfbf5f070a1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
@@ -11,7 +11,6 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.FixedBitSet;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.common.util.Maps;
@@ -133,15 +132,6 @@ private static boolean assertSearchCoordinationThread() {
     public void run() {
         assert assertSearchCoordinationThread();
         checkNoMissingShards();
-        Version version = request.minCompatibleShardNode();
-        if (version != null && Version.CURRENT.minimumCompatibilityVersion().equals(version) == false) {
-            if (checkMinimumVersion(shardsIts) == false) {
-                throw new VersionMismatchException(
-                    "One of the shards is incompatible with the required minimum version [{}]",
-                    request.minCompatibleShardNode()
-                );
-            }
-        }
         runCoordinatorRewritePhase();
     }
 
@@ -378,21 +368,6 @@ public CanMatchNodeRequest.Shard buildShardLevelRequest(SearchShardIterator shar
         );
     }
 
-    private boolean checkMinimumVersion(GroupShardsIterator<SearchShardIterator> shardsIts) {
-        for (SearchShardIterator it : shardsIts) {
-            if (it.getTargetNodeIds().isEmpty() == false) {
-                boolean isCompatible = it.getTargetNodeIds().stream().anyMatch(nodeId -> {
-                    Transport.Connection conn = getConnection(new SendingTarget(it.getClusterAlias(), nodeId));
-                    return conn == null || conn.getNode().getVersion().onOrAfter(request.minCompatibleShardNode());
-                });
-                if (isCompatible == false) {
-                    return false;
-                }
-            }
-        }
-        return true;
-    }
-
     @Override
     public void start() {
         if (getNumShards() == 0) {
@@ -421,12 +396,7 @@ public void onPhaseFailure(String msg, Exception cause) {
     }
 
     public Transport.Connection getConnection(SendingTarget sendingTarget) {
-        Transport.Connection conn = nodeIdToConnection.apply(sendingTarget.clusterAlias, sendingTarget.nodeId);
-        Version minVersion = request.minCompatibleShardNode();
-        if (minVersion != null && conn != null && conn.getNode().getVersion().before(minVersion)) {
-            throw new VersionMismatchException("One of the shards is incompatible with the required minimum version [{}]", minVersion);
-        }
-        return conn;
+        return nodeIdToConnection.apply(sendingTarget.clusterAlias, sendingTarget.nodeId);
     }
 
     private int getNumShards() {
diff --git a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java
index d41a2561646b8..b52d76aac4132 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java
@@ -57,8 +57,8 @@ public void consumeResult(SearchPhaseResult result, Runnable next) {
             return;
         }
         // set the relation to the first non-equal relation
-        relationAtomicReference.compareAndSet(TotalHits.Relation.EQUAL_TO, result.queryResult().getTotalHits().relation);
-        totalHits.add(result.queryResult().getTotalHits().value);
+        relationAtomicReference.compareAndSet(TotalHits.Relation.EQUAL_TO, result.queryResult().getTotalHits().relation());
+        totalHits.add(result.queryResult().getTotalHits().value());
         terminatedEarly.compareAndSet(false, (result.queryResult().terminatedEarly() != null && result.queryResult().terminatedEarly()));
         timedOut.compareAndSet(false, result.queryResult().searchTimedOut());
         next.run();
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 74786dff1648d..ca9c4ab44c423 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -892,8 +892,8 @@ TotalHits getTotalHits() {
 
         void add(TopDocsAndMaxScore topDocs, boolean timedOut, Boolean terminatedEarly) {
             if (trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
-                totalHits += topDocs.topDocs.totalHits.value;
-                if (topDocs.topDocs.totalHits.relation == Relation.GREATER_THAN_OR_EQUAL_TO) {
+                totalHits += topDocs.topDocs.totalHits.value();
+                if (topDocs.topDocs.totalHits.relation() == Relation.GREATER_THAN_OR_EQUAL_TO) {
                     totalHitsRelation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 9961c3770fa86..2e1d58e042f09 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.Rewriteable;
@@ -92,9 +93,6 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
 
     private boolean ccsMinimizeRoundtrips;
 
-    @Nullable
-    private final Version minCompatibleShardNode;
-
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
@@ -112,15 +110,10 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
     private boolean forceSyntheticSource = false;
 
     public SearchRequest() {
-        this((Version) null);
-    }
-
-    public SearchRequest(Version minCompatibleShardNode) {
         this.localClusterAlias = null;
         this.absoluteStartMillis = DEFAULT_ABSOLUTE_START_MILLIS;
         this.finalReduce = true;
-        this.minCompatibleShardNode = minCompatibleShardNode;
-        this.ccsMinimizeRoundtrips = minCompatibleShardNode == null;
+        this.ccsMinimizeRoundtrips = true;
     }
 
     /**
@@ -219,7 +212,6 @@ private SearchRequest(
         this.localClusterAlias = localClusterAlias;
         this.absoluteStartMillis = absoluteStartMillis;
         this.finalReduce = finalReduce;
-        this.minCompatibleShardNode = searchRequest.minCompatibleShardNode;
         this.waitForCheckpoints = searchRequest.waitForCheckpoints;
         this.waitForCheckpointsTimeout = searchRequest.waitForCheckpointsTimeout;
         this.forceSyntheticSource = searchRequest.forceSyntheticSource;
@@ -263,10 +255,10 @@ public SearchRequest(StreamInput in) throws IOException {
             finalReduce = true;
         }
         ccsMinimizeRoundtrips = in.readBoolean();
-        if (in.readBoolean()) {
-            minCompatibleShardNode = Version.readVersion(in);
-        } else {
-            minCompatibleShardNode = null;
+        if ((in.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)
+            || in.getTransportVersion().onOrAfter(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE)) && in.readBoolean()) {
+            @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)  // this can be removed (again) when the v9 transport version can diverge
+            Version v = Version.readVersion(in); // and drop on the floor
         }
         waitForCheckpoints = in.readMap(StreamInput::readLongArray);
         waitForCheckpointsTimeout = in.readTimeValue();
@@ -302,9 +294,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(finalReduce);
         }
         out.writeBoolean(ccsMinimizeRoundtrips);
-        out.writeBoolean(minCompatibleShardNode != null);
-        if (minCompatibleShardNode != null) {
-            Version.writeVersion(minCompatibleShardNode, out);
+        if (out.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)
+            || out.getTransportVersion().onOrAfter(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE)) {
+            out.writeBoolean(false);
         }
         out.writeMap(waitForCheckpoints, StreamOutput::writeLongArray);
         out.writeTimeValue(waitForCheckpointsTimeout);
@@ -351,14 +343,6 @@ public ActionRequestValidationException validate() {
                 validationException = addValidationError("[preference] cannot be used with point in time", validationException);
             }
         }
-        if (minCompatibleShardNode() != null) {
-            if (isCcsMinimizeRoundtrips()) {
-                validationException = addValidationError(
-                    "[ccs_minimize_roundtrips] cannot be [true] when setting a minimum compatible " + "shard version",
-                    validationException
-                );
-            }
-        }
         if (pointInTimeBuilder() != null && waitForCheckpoints.isEmpty() == false) {
             validationException = addValidationError("using [point in time] is not allowed with wait_for_checkpoints", validationException);
 
@@ -401,15 +385,6 @@ long getAbsoluteStartMillis() {
         return absoluteStartMillis;
     }
 
-    /**
-     * Returns the minimum compatible shard version the search request needs to run on. If the version is null, then there are no
-     * restrictions imposed on shards versions part of this search.
-     */
-    @Nullable
-    public Version minCompatibleShardNode() {
-        return minCompatibleShardNode;
-    }
-
     /**
      * Sets the indices the search will be executed on.
      */
@@ -818,7 +793,6 @@ public boolean equals(Object o) {
             && Objects.equals(localClusterAlias, that.localClusterAlias)
             && absoluteStartMillis == that.absoluteStartMillis
             && ccsMinimizeRoundtrips == that.ccsMinimizeRoundtrips
-            && Objects.equals(minCompatibleShardNode, that.minCompatibleShardNode)
             && forceSyntheticSource == that.forceSyntheticSource;
     }
 
@@ -840,7 +814,6 @@ public int hashCode() {
             localClusterAlias,
             absoluteStartMillis,
             ccsMinimizeRoundtrips,
-            minCompatibleShardNode,
             forceSyntheticSource
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 83ee6c216ad49..041b3ae73c1ee 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -787,7 +787,8 @@ public boolean hasClusterObjects() {
          *              This will be false for local-cluster (non-CCS) only searches.
          */
         public boolean hasRemoteClusters() {
-            return total > 1 || clusterInfo.keySet().stream().anyMatch(alias -> alias != RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+            return total > 1
+                || clusterInfo.keySet().stream().anyMatch(alias -> alias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false);
         }
 
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 1645a378446a4..302c3e243a1f6 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -1247,6 +1247,29 @@ private void executeSearch(
                 indicesAndAliases,
                 concreteLocalIndices
             );
+
+            // localShardIterators is empty since there are no matching indices. In such cases,
+            // we update the local cluster's status from RUNNING to SUCCESSFUL right away. Before
+            // we attempt to do that, we must ensure that the local cluster was specified in the user's
+            // search request. This is done by trying to fetch the local cluster via getCluster() and
+            // checking for a non-null return value. If the local cluster was never specified, its status
+            // update can be skipped.
+            if (localShardIterators.isEmpty()
+                && clusters != SearchResponse.Clusters.EMPTY
+                && clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) != null) {
+                clusters.swapCluster(
+                    RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                    (alias, v) -> new SearchResponse.Cluster.Builder(v).setStatus(SearchResponse.Cluster.Status.SUCCESSFUL)
+                        .setTotalShards(0)
+                        .setSuccessfulShards(0)
+                        .setSkippedShards(0)
+                        .setFailedShards(0)
+                        .setFailures(Collections.emptyList())
+                        .setTook(TimeValue.timeValueMillis(0))
+                        .setTimedOut(false)
+                        .build()
+                );
+            }
         }
         final GroupShardsIterator<SearchShardIterator> shardIterators = mergeShardsIterators(localShardIterators, remoteShardIterators);
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java
new file mode 100644
index 0000000000000..910be151d1bf5
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * We define as index components the two different sets of indices a data stream could consist of:
+ * - DATA: represents the backing indices
+ * - FAILURES: represent the failing indices
+ * - ALL: represents all available in this expression components, meaning if it's a data stream both backing and failure indices and if it's
+ * an index only the index itself.
+ * Note: An index is its own DATA component, but it cannot have a FAILURE component.
+ */
+public enum IndexComponentSelector implements Writeable {
+    DATA("data", (byte) 0),
+    FAILURES("failures", (byte) 1),
+    ALL_APPLICABLE("*", (byte) 2);
+
+    private final String key;
+    private final byte id;
+
+    IndexComponentSelector(String key, byte id) {
+        this.key = key;
+        this.id = id;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public byte getId() {
+        return id;
+    }
+
+    private static final Map<String, IndexComponentSelector> KEY_REGISTRY;
+    private static final Map<Byte, IndexComponentSelector> ID_REGISTRY;
+
+    static {
+        Map<String, IndexComponentSelector> keyRegistry = new HashMap<>(IndexComponentSelector.values().length);
+        for (IndexComponentSelector value : IndexComponentSelector.values()) {
+            keyRegistry.put(value.getKey(), value);
+        }
+        KEY_REGISTRY = Collections.unmodifiableMap(keyRegistry);
+        Map<Byte, IndexComponentSelector> idRegistry = new HashMap<>(IndexComponentSelector.values().length);
+        for (IndexComponentSelector value : IndexComponentSelector.values()) {
+            idRegistry.put(value.getId(), value);
+        }
+        ID_REGISTRY = Collections.unmodifiableMap(idRegistry);
+    }
+
+    /**
+     * Retrieves the respective selector when the suffix key is recognised
+     * @param key the suffix key, probably parsed from an expression
+     * @return the selector or null if the key was not recognised.
+     */
+    @Nullable
+    public static IndexComponentSelector getByKey(String key) {
+        return KEY_REGISTRY.get(key);
+    }
+
+    public static IndexComponentSelector read(StreamInput in) throws IOException {
+        return getById(in.readByte());
+    }
+
+    // Visible for testing
+    static IndexComponentSelector getById(byte id) {
+        IndexComponentSelector indexComponentSelector = ID_REGISTRY.get(id);
+        if (indexComponentSelector == null) {
+            throw new IllegalArgumentException(
+                "Unknown id of index component selector [" + id + "], available options are: " + ID_REGISTRY
+            );
+        }
+        return indexComponentSelector;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeByte(id);
+    }
+
+    public boolean shouldIncludeData() {
+        return this == ALL_APPLICABLE || this == DATA;
+    }
+
+    public boolean shouldIncludeFailures() {
+        return this == ALL_APPLICABLE || this == FAILURES;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index d3ea063247704..85889d8398cb1 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -31,7 +31,6 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
@@ -48,24 +47,36 @@
  * @param gatekeeperOptions, applies to all the resolved indices and defines if throttled will be included and if certain type of
  *                        aliases or indices are allowed, or they will throw an error. It acts as a gatekeeper when an action
  *                        does not support certain options.
- * @param failureStoreOptions, applies to all indices already matched and controls the type of indices that will be returned. Currently,
- *                             there are two types, data stream failure indices (only certain data streams have them) and data stream
- *                             backing indices or stand-alone indices.
+ * @param selectorOptions, applies to all resolved expressions, and it specifies the index component that should be included, if there
+ *                         is no index component defined on the expression level.
  */
 public record IndicesOptions(
     ConcreteTargetOptions concreteTargetOptions,
     WildcardOptions wildcardOptions,
     GatekeeperOptions gatekeeperOptions,
-    FailureStoreOptions failureStoreOptions
+    SelectorOptions selectorOptions
 ) implements ToXContentFragment {
 
-    public IndicesOptions(
-        ConcreteTargetOptions concreteTargetOptions,
-        WildcardOptions wildcardOptions,
-        GatekeeperOptions gatekeeperOptions
-    ) {
-        this(concreteTargetOptions, wildcardOptions, gatekeeperOptions, FailureStoreOptions.DEFAULT);
-    }
+    /**
+     * @deprecated this query param will be replaced by the selector `::` on the expression level
+     */
+    @Deprecated
+    public static final String FAILURE_STORE_QUERY_PARAM = "failure_store";
+    /**
+     * @deprecated this value will be replaced by the selector `::*` on the expression level
+     */
+    @Deprecated
+    public static final String INCLUDE_ALL = "include";
+    /**
+     * @deprecated this value will be replaced by the selector `::data` on the expression level
+     */
+    @Deprecated
+    public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude";
+    /**
+     * @deprecated this value will be replaced by the selector `::failures` on the expression level
+     */
+    @Deprecated
+    public static final String INCLUDE_ONLY_FAILURE_INDICES = "only";
 
     public static IndicesOptions.Builder builder() {
         return new Builder();
@@ -310,7 +321,7 @@ public static Builder builder(WildcardOptions wildcardOptions) {
      * - The "allow*" flags, which purpose is to enable actions to define certain conditions that need to apply on the concrete indices
      * they accept. For example, single-index actions will set allowAliasToMultipleIndices to false, while search will not accept a
      * closed index etc. These options are not configurable by the end-user.
-     * - The ignoreThrottled flag, which is a depricared flag that will filter out frozen indices.
+     * - The ignoreThrottled flag, which is a deprecated flag that will filter out frozen indices.
      * @param allowAliasToMultipleIndices, allow aliases to multiple indices, true by default.
      * @param allowClosedIndices, allow closed indices, true by default.
      * @param allowFailureIndices, allow failure indices in the response, true by default
@@ -408,97 +419,47 @@ public static Builder builder(GatekeeperOptions gatekeeperOptions) {
     }
 
     /**
-     * Applies to all indices already matched and controls the type of indices that will be returned. There are two types, data stream
-     * failure indices (only certain data streams have them) and data stream backing indices or stand-alone indices.
-     * @param includeRegularIndices, when true regular or data stream backing indices will be retrieved.
-     * @param includeFailureIndices, when true data stream failure indices will be included.
+     * Defines which selectors should be used by default for an index operation in the event that no selectors are provided.
      */
-    public record FailureStoreOptions(boolean includeRegularIndices, boolean includeFailureIndices)
-        implements
-            Writeable,
-            ToXContentFragment {
-
-        public static final String FAILURE_STORE = "failure_store";
-        public static final String INCLUDE_ALL = "include";
-        public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude";
-        public static final String INCLUDE_ONLY_FAILURE_INDICES = "only";
-
-        public static final FailureStoreOptions DEFAULT = new FailureStoreOptions(true, false);
-
-        public static FailureStoreOptions read(StreamInput in) throws IOException {
-            return new FailureStoreOptions(in.readBoolean(), in.readBoolean());
-        }
+    public record SelectorOptions(IndexComponentSelector defaultSelector) implements Writeable {
 
-        public static FailureStoreOptions parseParameters(Object failureStoreValue, FailureStoreOptions defaultOptions) {
-            if (failureStoreValue == null) {
-                return defaultOptions;
-            }
-            FailureStoreOptions.Builder builder = defaultOptions == null
-                ? new FailureStoreOptions.Builder()
-                : new FailureStoreOptions.Builder(defaultOptions);
-            return switch (failureStoreValue.toString()) {
-                case INCLUDE_ALL -> builder.includeRegularIndices(true).includeFailureIndices(true).build();
-                case INCLUDE_ONLY_REGULAR_INDICES -> builder.includeRegularIndices(true).includeFailureIndices(false).build();
-                case INCLUDE_ONLY_FAILURE_INDICES -> builder.includeRegularIndices(false).includeFailureIndices(true).build();
-                default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE + " value [" + failureStoreValue + "]");
-            };
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder.field(FAILURE_STORE, displayValue());
-        }
-
-        public String displayValue() {
-            if (includeRegularIndices && includeFailureIndices) {
-                return INCLUDE_ALL;
-            } else if (includeRegularIndices) {
-                return INCLUDE_ONLY_REGULAR_INDICES;
+        public static final SelectorOptions ALL_APPLICABLE = new SelectorOptions(IndexComponentSelector.ALL_APPLICABLE);
+        public static final SelectorOptions DATA = new SelectorOptions(IndexComponentSelector.DATA);
+        public static final SelectorOptions FAILURES = new SelectorOptions(IndexComponentSelector.FAILURES);
+        /**
+         * Default instance. Uses <pre>::data</pre> as the default selector if none are present in an index expression.
+         */
+        public static final SelectorOptions DEFAULT = DATA;
+
+        public static SelectorOptions read(StreamInput in) throws IOException {
+            if (in.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
+                EnumSet<IndexComponentSelector> set = in.readEnumSet(IndexComponentSelector.class);
+                if (set.isEmpty() || set.size() == 2) {
+                    assert set.contains(IndexComponentSelector.DATA) && set.contains(IndexComponentSelector.FAILURES)
+                        : "The enum set only supported ::data and ::failures";
+                    return SelectorOptions.ALL_APPLICABLE;
+                } else if (set.contains(IndexComponentSelector.DATA)) {
+                    return SelectorOptions.DATA;
+                } else {
+                    return SelectorOptions.FAILURES;
+                }
+            } else {
+                return new SelectorOptions(IndexComponentSelector.read(in));
             }
-            return INCLUDE_ONLY_FAILURE_INDICES;
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeBoolean(includeRegularIndices);
-            out.writeBoolean(includeFailureIndices);
-        }
-
-        public static class Builder {
-            private boolean includeRegularIndices;
-            private boolean includeFailureIndices;
-
-            public Builder() {
-                this(DEFAULT);
-            }
-
-            Builder(FailureStoreOptions options) {
-                includeRegularIndices = options.includeRegularIndices;
-                includeFailureIndices = options.includeFailureIndices;
-            }
-
-            public Builder includeRegularIndices(boolean includeRegularIndices) {
-                this.includeRegularIndices = includeRegularIndices;
-                return this;
-            }
-
-            public Builder includeFailureIndices(boolean includeFailureIndices) {
-                this.includeFailureIndices = includeFailureIndices;
-                return this;
-            }
-
-            public FailureStoreOptions build() {
-                return new FailureStoreOptions(includeRegularIndices, includeFailureIndices);
+            if (out.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
+                switch (defaultSelector) {
+                    case ALL_APPLICABLE -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES));
+                    case DATA -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA));
+                    case FAILURES -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.FAILURES));
+                }
+            } else {
+                defaultSelector.writeTo(out);
             }
         }
-
-        public static Builder builder() {
-            return new Builder();
-        }
-
-        public static Builder builder(FailureStoreOptions failureStoreOptions) {
-            return new Builder(failureStoreOptions);
-        }
     }
 
     /**
@@ -550,7 +511,7 @@ private enum Option {
         ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
         WildcardOptions.DEFAULT,
         GatekeeperOptions.DEFAULT,
-        FailureStoreOptions.DEFAULT
+        SelectorOptions.DEFAULT
     );
 
     public static final IndicesOptions STRICT_EXPAND_OPEN = IndicesOptions.builder()
@@ -570,7 +531,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_FAILURE_STORE = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -589,7 +550,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -608,7 +569,25 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
+        .build();
+    public static final IndicesOptions LENIENT_EXPAND_OPEN_NO_SELECTORS = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder()
+                .matchOpen(true)
+                .matchClosed(false)
+                .includeHidden(false)
+                .allowEmptyExpressions(true)
+                .resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .allowFailureIndices(false)
+                .ignoreThrottled(false)
+        )
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_HIDDEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -627,7 +606,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -646,7 +625,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -660,7 +639,20 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
+        .build();
+    public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder().matchOpen(true).matchClosed(true).includeHidden(true).allowEmptyExpressions(true).resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .allowFailureIndices(false)
+                .ignoreThrottled(false)
+        )
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -679,7 +671,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -693,7 +685,20 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
+        .build();
+    public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTORS = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder().matchOpen(true).matchClosed(true).includeHidden(true).allowEmptyExpressions(true).resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .allowFailureIndices(false)
+                .ignoreThrottled(false)
+        )
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -712,7 +717,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -726,7 +731,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -745,7 +750,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -764,7 +769,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -783,7 +788,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -802,7 +807,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .allowAliasToMultipleIndices(true)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -821,7 +826,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -840,7 +845,7 @@ private enum Option {
                 .allowFailureIndices(true)
                 .ignoreThrottled(false)
         )
-        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
+        .selectorOptions(SelectorOptions.DATA)
         .build();
 
     /**
@@ -898,7 +903,7 @@ public boolean forbidClosedIndices() {
     }
 
     /**
-     * @return Whether execution on closed indices is allowed.
+     * @return Whether execution on failure indices is allowed.
      */
     public boolean allowFailureIndices() {
         return gatekeeperOptions.allowFailureIndices();
@@ -929,14 +934,14 @@ public boolean ignoreThrottled() {
      * @return whether regular indices (stand-alone or backing indices) will be included in the response
      */
     public boolean includeRegularIndices() {
-        return failureStoreOptions().includeRegularIndices();
+        return selectorOptions().defaultSelector().shouldIncludeData();
     }
 
     /**
      * @return whether failure indices (only supported by certain data streams) will be included in the response
      */
     public boolean includeFailureIndices() {
-        return failureStoreOptions().includeFailureIndices();
+        return selectorOptions().defaultSelector().shouldIncludeFailures();
     }
 
     public void writeIndicesOptions(StreamOutput out) throws IOException {
@@ -977,8 +982,13 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
             states.add(WildcardStates.HIDDEN);
         }
         out.writeEnumSet(states);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
-            failureStoreOptions.writeTo(out);
+        if (out.getTransportVersion()
+            .between(TransportVersions.V_8_14_0, TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) {
+            out.writeBoolean(includeRegularIndices());
+            out.writeBoolean(includeFailureIndices());
+        }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) {
+            selectorOptions.writeTo(out);
         }
     }
 
@@ -999,16 +1009,30 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
             .allowFailureIndices(allowFailureIndices)
             .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED))
             .build();
-        FailureStoreOptions failureStoreOptions = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
-            ? FailureStoreOptions.read(in)
-            : FailureStoreOptions.DEFAULT;
+        SelectorOptions selectorOptions = SelectorOptions.DEFAULT;
+        if (in.getTransportVersion()
+            .between(TransportVersions.V_8_14_0, TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) {
+            // Reading from an older node, which will be sending two booleans that we must read out and ignore.
+            var includeData = in.readBoolean();
+            var includeFailures = in.readBoolean();
+            if (includeData && includeFailures) {
+                selectorOptions = SelectorOptions.ALL_APPLICABLE;
+            } else if (includeData) {
+                selectorOptions = SelectorOptions.DATA;
+            } else {
+                selectorOptions = SelectorOptions.FAILURES;
+            }
+        }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) {
+            selectorOptions = SelectorOptions.read(in);
+        }
         return new IndicesOptions(
             options.contains(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS)
                 ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS
                 : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
             wildcardOptions,
             gatekeeperOptions,
-            failureStoreOptions
+            selectorOptions
         );
     }
 
@@ -1016,7 +1040,7 @@ public static class Builder {
         private ConcreteTargetOptions concreteTargetOptions;
         private WildcardOptions wildcardOptions;
         private GatekeeperOptions gatekeeperOptions;
-        private FailureStoreOptions failureStoreOptions;
+        private SelectorOptions selectorOptions;
 
         Builder() {
             this(DEFAULT);
@@ -1026,7 +1050,7 @@ public static class Builder {
             concreteTargetOptions = indicesOptions.concreteTargetOptions;
             wildcardOptions = indicesOptions.wildcardOptions;
             gatekeeperOptions = indicesOptions.gatekeeperOptions;
-            failureStoreOptions = indicesOptions.failureStoreOptions;
+            selectorOptions = indicesOptions.selectorOptions;
         }
 
         public Builder concreteTargetOptions(ConcreteTargetOptions concreteTargetOptions) {
@@ -1054,25 +1078,13 @@ public Builder gatekeeperOptions(GatekeeperOptions.Builder generalOptions) {
             return this;
         }
 
-        public Builder failureStoreOptions(FailureStoreOptions failureStoreOptions) {
-            this.failureStoreOptions = failureStoreOptions;
-            return this;
-        }
-
-        public Builder failureStoreOptions(FailureStoreOptions.Builder failureStoreOptions) {
-            this.failureStoreOptions = failureStoreOptions.build();
-            return this;
-        }
-
-        public Builder failureStoreOptions(Consumer<FailureStoreOptions.Builder> failureStoreOptionsConfig) {
-            FailureStoreOptions.Builder failureStoreOptionsBuilder = FailureStoreOptions.builder(failureStoreOptions);
-            failureStoreOptionsConfig.accept(failureStoreOptionsBuilder);
-            this.failureStoreOptions = failureStoreOptionsBuilder.build();
+        public Builder selectorOptions(SelectorOptions selectorOptions) {
+            this.selectorOptions = selectorOptions;
             return this;
         }
 
         public IndicesOptions build() {
-            return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, failureStoreOptions);
+            return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions);
         }
     }
 
@@ -1171,11 +1183,12 @@ public static IndicesOptions fromOptions(
             .allowClosedIndices(forbidClosedIndices == false)
             .ignoreThrottled(ignoreThrottled)
             .build();
+        final SelectorOptions selectorOptions = SelectorOptions.DEFAULT;
         return new IndicesOptions(
             ignoreUnavailable ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
             wildcards,
             gatekeeperOptions,
-            FailureStoreOptions.DEFAULT
+            selectorOptions
         );
     }
 
@@ -1189,9 +1202,7 @@ public static IndicesOptions fromRequest(RestRequest request, IndicesOptions def
             request.param(ConcreteTargetOptions.IGNORE_UNAVAILABLE),
             request.param(WildcardOptions.ALLOW_NO_INDICES),
             request.param(GatekeeperOptions.IGNORE_THROTTLED),
-            DataStream.isFailureStoreFeatureFlagEnabled()
-                ? request.param(FailureStoreOptions.FAILURE_STORE)
-                : FailureStoreOptions.INCLUDE_ONLY_REGULAR_INDICES,
+            DataStream.isFailureStoreFeatureFlagEnabled() ? request.param(FAILURE_STORE_QUERY_PARAM) : INCLUDE_ONLY_REGULAR_INDICES,
             defaultSettings
         );
     }
@@ -1207,7 +1218,7 @@ public static IndicesOptions fromMap(Map<String, Object> map, IndicesOptions def
                 map.containsKey(GatekeeperOptions.IGNORE_THROTTLED)
                     ? map.get(GatekeeperOptions.IGNORE_THROTTLED)
                     : map.get("ignoreThrottled"),
-                map.containsKey(FailureStoreOptions.FAILURE_STORE) ? map.get(FailureStoreOptions.FAILURE_STORE) : map.get("failureStore"),
+                map.containsKey(FAILURE_STORE_QUERY_PARAM) ? map.get(FAILURE_STORE_QUERY_PARAM) : map.get("failureStore"),
                 defaultSettings
             );
         }
@@ -1235,7 +1246,7 @@ public static boolean isIndicesOptions(String name) {
             || "ignoreThrottled".equals(name)
             || WildcardOptions.ALLOW_NO_INDICES.equals(name)
             || "allowNoIndices".equals(name)
-            || (DataStream.isFailureStoreFeatureFlagEnabled() && FailureStoreOptions.FAILURE_STORE.equals(name))
+            || (DataStream.isFailureStoreFeatureFlagEnabled() && FAILURE_STORE_QUERY_PARAM.equals(name))
             || (DataStream.isFailureStoreFeatureFlagEnabled() && "failureStore".equals(name));
     }
 
@@ -1267,26 +1278,51 @@ public static IndicesOptions fromParameters(
 
         WildcardOptions wildcards = WildcardOptions.parseParameters(wildcardsString, allowNoIndicesString, defaultSettings.wildcardOptions);
         GatekeeperOptions gatekeeperOptions = GatekeeperOptions.parseParameter(ignoreThrottled, defaultSettings.gatekeeperOptions);
-        FailureStoreOptions failureStoreOptions = DataStream.isFailureStoreFeatureFlagEnabled()
-            ? FailureStoreOptions.parseParameters(failureStoreString, defaultSettings.failureStoreOptions)
-            : FailureStoreOptions.DEFAULT;
+        SelectorOptions selectorOptions = DataStream.isFailureStoreFeatureFlagEnabled()
+            ? parseFailureStoreParameters(failureStoreString, defaultSettings.selectorOptions)
+            : SelectorOptions.DEFAULT;
 
         // note that allowAliasesToMultipleIndices is not exposed, always true (only for internal use)
         return IndicesOptions.builder()
             .concreteTargetOptions(ConcreteTargetOptions.fromParameter(ignoreUnavailableString, defaultSettings.concreteTargetOptions))
             .wildcardOptions(wildcards)
             .gatekeeperOptions(gatekeeperOptions)
-            .failureStoreOptions(failureStoreOptions)
+            .selectorOptions(selectorOptions)
             .build();
     }
 
+    /**
+     * @deprecated This method parses the query parameter failure_store. This is a deprecated param, and it will be replaced
+     * the selector suffix, for example `my-data-stream::data` or `my-data-stream::failures`
+     */
+    @Deprecated
+    private static SelectorOptions parseFailureStoreParameters(Object failureStoreValue, SelectorOptions defaultOptions) {
+        if (failureStoreValue == null) {
+            return defaultOptions;
+        }
+        return switch (failureStoreValue.toString()) {
+            case INCLUDE_ALL -> SelectorOptions.ALL_APPLICABLE;
+            case INCLUDE_ONLY_REGULAR_INDICES -> SelectorOptions.DATA;
+            case INCLUDE_ONLY_FAILURE_INDICES -> SelectorOptions.FAILURES;
+            default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE_QUERY_PARAM + " value [" + failureStoreValue + "]");
+        };
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         concreteTargetOptions.toXContent(builder, params);
         wildcardOptions.toXContent(builder, params);
         gatekeeperOptions.toXContent(builder, params);
         if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-            failureStoreOptions.toXContent(builder, params);
+            String displayValue;
+            if (SelectorOptions.ALL_APPLICABLE.equals(selectorOptions())) {
+                displayValue = INCLUDE_ALL;
+            } else if (SelectorOptions.DATA.equals(selectorOptions())) {
+                displayValue = INCLUDE_ONLY_REGULAR_INDICES;
+            } else {
+                displayValue = INCLUDE_ONLY_FAILURE_INDICES;
+            }
+            builder.field(FAILURE_STORE_QUERY_PARAM, displayValue);
         }
         return builder;
     }
@@ -1295,7 +1331,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
     private static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField(ConcreteTargetOptions.IGNORE_UNAVAILABLE);
     private static final ParseField IGNORE_THROTTLED_FIELD = new ParseField(GatekeeperOptions.IGNORE_THROTTLED).withAllDeprecated();
     private static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField(WildcardOptions.ALLOW_NO_INDICES);
-    private static final ParseField FAILURE_STORE_FIELD = new ParseField(FailureStoreOptions.FAILURE_STORE);
+    private static final ParseField FAILURE_STORE_FIELD = new ParseField(FAILURE_STORE_QUERY_PARAM);
 
     public static IndicesOptions fromXContent(XContentParser parser) throws IOException {
         return fromXContent(parser, null);
@@ -1306,7 +1342,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
         WildcardOptions.Builder wildcards = defaults == null ? null : WildcardOptions.builder(defaults.wildcardOptions());
         GatekeeperOptions.Builder generalOptions = GatekeeperOptions.builder()
             .ignoreThrottled(defaults != null && defaults.gatekeeperOptions().ignoreThrottled());
-        FailureStoreOptions failureStoreOptions = defaults == null ? FailureStoreOptions.DEFAULT : defaults.failureStoreOptions();
+        SelectorOptions selectorOptions = defaults == null ? SelectorOptions.DEFAULT : defaults.selectorOptions();
         Boolean allowNoIndices = defaults == null ? null : defaults.allowNoIndices();
         Boolean ignoreUnavailable = defaults == null ? null : defaults.ignoreUnavailable();
         Token token = parser.currentToken() == Token.START_OBJECT ? parser.currentToken() : parser.nextToken();
@@ -1358,7 +1394,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
                     generalOptions.ignoreThrottled(parser.booleanValue());
                 } else if (DataStream.isFailureStoreFeatureFlagEnabled()
                     && FAILURE_STORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        failureStoreOptions = FailureStoreOptions.parseParameters(parser.text(), failureStoreOptions);
+                        selectorOptions = parseFailureStoreParameters(parser.text(), selectorOptions);
                     } else {
                         throw new ElasticsearchParseException(
                             "could not read indices options. Unexpected index option [" + currentFieldName + "]"
@@ -1389,7 +1425,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
             .concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable))
             .wildcardOptions(wildcards)
             .gatekeeperOptions(generalOptions)
-            .failureStoreOptions(failureStoreOptions)
+            .selectorOptions(selectorOptions)
             .build();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
index 2566e109b70e4..8d29025c1f0be 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
@@ -78,7 +78,7 @@ protected MasterNodeRequest(TimeValue masterNodeTimeout) {
     protected MasterNodeRequest(StreamInput in) throws IOException {
         super(in);
         masterNodeTimeout = in.readTimeValue();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             masterTerm = in.readVLong();
         } else {
             masterTerm = 0L;
@@ -92,7 +92,7 @@ public void writeTo(StreamOutput out) throws IOException {
         assert masterTerm <= newMasterTerm : masterTerm + " vs " + newMasterTerm;
         super.writeTo(out);
         out.writeTimeValue(masterNodeTimeout);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeVLong(newMasterTerm);
         } // else no protection against routing loops in older versions
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java
index 7414aeeb2c405..683c3589c893d 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.translog.Translog;
@@ -52,7 +53,9 @@ public void refreshShard(
             case WAIT_UNTIL -> waitUntil(indexShard, location, new ActionListener<>() {
                 @Override
                 public void onResponse(Boolean forced) {
-                    if (location != null && indexShard.routingEntry().isSearchable() == false) {
+                    // Fast refresh indices do not depend on the unpromotables being refreshed
+                    boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings());
+                    if (location != null && (indexShard.routingEntry().isSearchable() == false && fastRefresh == false)) {
                         refreshUnpromotables(indexShard, location, listener, forced, postWriteRefreshTimeout);
                     } else {
                         listener.onResponse(forced);
@@ -65,7 +68,9 @@ public void onFailure(Exception e) {
                 }
             });
             case IMMEDIATE -> immediate(indexShard, listener.delegateFailureAndWrap((l, r) -> {
-                if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0) {
+                // Fast refresh indices do not depend on the unpromotables being refreshed
+                boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings());
+                if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0 && fastRefresh == false) {
                     sendUnpromotableRequests(indexShard, r.generation(), true, l, postWriteRefreshTimeout);
                 } else {
                     l.onResponse(true);
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index 2beeaf1a26f0f..42196fac28528 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -16,8 +16,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -110,9 +108,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.startObject();
                 Failure failure = response.getFailure();
                 builder.field(Fields._INDEX, failure.getIndex());
-                if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                    builder.field(Fields._TYPE, MapperService.SINGLE_MAPPING_NAME);
-                }
                 builder.field(Fields._ID, failure.getId());
                 ElasticsearchException.generateFailureXContent(builder, params, failure.getCause(), true);
                 builder.endObject();
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
index 2d70af94d53d1..4c4aa7de46f6b 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -164,9 +163,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (isArtificial() == false) {
             builder.field(FieldStrings._ID, id);
         }
-        if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-            builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-        }
         builder.field(FieldStrings._VERSION, docVersion);
         builder.field(FieldStrings.FOUND, isExists());
         builder.field(FieldStrings.TOOK, tookInMillis);
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index 021ad8127a2d0..6a881163914e4 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -701,7 +701,7 @@ String jvmVendor() {
         }
 
         String javaVersion() {
-            return Constants.JAVA_VERSION;
+            return Runtime.version().toString();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java
index cb98cd4b2f535..ac96a2d55bc71 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java
@@ -228,7 +228,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
             builder.endObject();
 
-            builder.timeField("oldest_start_time_millis", "oldest_start_time", firstStartTimeMillis);
+            builder.timestampFieldsFromUnixEpochMillis("oldest_start_time_millis", "oldest_start_time", firstStartTimeMillis);
 
             return builder.endObject();
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 64df6e77326e4..f7cad013554c6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -241,9 +241,7 @@ public ClusterState(
     }
 
     private boolean assertEventIngestedIsUnknownInMixedClusters(Metadata metadata, CompatibilityVersions compatibilityVersions) {
-        if (compatibilityVersions.transportVersion().before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)
-            && metadata != null
-            && metadata.indices() != null) {
+        if (compatibilityVersions.transportVersion().before(TransportVersions.V_8_15_0) && metadata != null && metadata.indices() != null) {
             for (IndexMetadata indexMetadata : metadata.indices().values()) {
                 assert indexMetadata.getEventIngestedRange() == IndexLongFieldRange.UNKNOWN
                     : "event.ingested range should be UNKNOWN but is "
diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
index c371ff4d37a05..fe144135d42bd 100644
--- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
@@ -180,7 +180,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignore
                         builder.value(snapshot.getName());
                     }
                     builder.endArray();
-                    builder.timeField("start_time_millis", "start_time", entry.startTime);
+                    builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", entry.startTime);
                     builder.field("repository_state_id", entry.repositoryStateId);
                     builder.field("state", entry.state);
                 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
index c32175fc9367d..d82a31720d6d4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
@@ -1404,7 +1404,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 }
             }
             builder.endArray();
-            builder.timeField("start_time_millis", "start_time", startTime);
+            builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", startTime);
             builder.field("repository_state_id", repositoryStateId);
             builder.startArray("shards");
             {
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 4b84854315a82..bd438d66549aa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -722,7 +722,7 @@ public ClusterState execute(BatchExecutionContext<StartedShardUpdateTask> batchE
                              */
                             IndexLongFieldRange newEventIngestedMillisRange = IndexLongFieldRange.UNKNOWN;
                             TransportVersion minTransportVersion = batchExecutionContext.initialState().getMinTransportVersion();
-                            if (minTransportVersion.onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+                            if (minTransportVersion.onOrAfter(TransportVersions.V_8_15_0)) {
                                 newEventIngestedMillisRange = currentEventIngestedMillisRange.extendWithShardRange(
                                     startedShardEntry.shardId.id(),
                                     indexMetadata.getNumberOfShards(),
@@ -827,7 +827,7 @@ public static class StartedShardEntry extends TransportRequest {
             primaryTerm = in.readVLong();
             this.message = in.readString();
             this.timestampRange = ShardLongFieldRange.readFrom(in);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 this.eventIngestedRange = ShardLongFieldRange.readFrom(in);
             } else {
                 this.eventIngestedRange = ShardLongFieldRange.UNKNOWN;
@@ -858,7 +858,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(primaryTerm);
             out.writeString(message);
             timestampRange.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 eventIngestedRange.writeTo(out);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java
index 7b9690ad2e2b2..84f9c42d27ece 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -194,9 +193,7 @@ public void start() {
          * system context.
          */
         if (clusterService.localNode().isMasterNode() == false) {
-            final ThreadContext threadContext = transportService.getThreadPool().getThreadContext();
-            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
-                threadContext.markAsSystemContext();
+            try (var ignored = transportService.getThreadPool().getThreadContext().newEmptySystemContext()) {
                 beginPollingRemoteMasterStabilityDiagnostic();
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java
index 5182e01899c6c..89083848d488d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java
@@ -25,7 +25,7 @@ public JoinStatus(StreamInput in) throws IOException {
             new DiscoveryNode(in),
             in.readLong(),
             in.readString(),
-            in.getTransportVersion().onOrAfter(TransportVersions.JOIN_STATUS_AGE_SERIALIZATION)
+            in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)
                 ? in.readTimeValue()
                 : new TimeValue(in.readLong(), TimeUnit.valueOf(in.readString()))
         );
@@ -36,7 +36,7 @@ public void writeTo(StreamOutput out) throws IOException {
         remoteNode.writeTo(out);
         out.writeLong(term);
         out.writeString(message);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.JOIN_STATUS_AGE_SERIALIZATION)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeTimeValue(age);
         } else {
             out.writeLong(age.duration());
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java
index 34d59c9860aba..7de7fd4d92d1b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java
@@ -13,7 +13,6 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.cluster.ClusterState;
@@ -31,7 +30,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -46,7 +44,6 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -162,55 +159,14 @@ public void validateJoin(DiscoveryNode discoveryNode, ActionListener<Void> liste
             return;
         }
 
-        if (connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0)) {
-            if (executeRefs.tryIncRef()) {
-                try {
-                    execute(new JoinValidation(discoveryNode, connection, listener));
-                } finally {
-                    executeRefs.decRef();
-                }
-            } else {
-                listener.onFailure(new NodeClosedException(transportService.getLocalNode()));
+        if (executeRefs.tryIncRef()) {
+            try {
+                execute(new JoinValidation(discoveryNode, connection, listener));
+            } finally {
+                executeRefs.decRef();
             }
         } else {
-            legacyValidateJoin(discoveryNode, listener, connection);
-        }
-    }
-
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    private void legacyValidateJoin(DiscoveryNode discoveryNode, ActionListener<Void> listener, Transport.Connection connection) {
-        final var responseHandler = TransportResponseHandler.empty(responseExecutor, listener.delegateResponse((l, e) -> {
-            logger.warn(() -> "failed to validate incoming join request from node [" + discoveryNode + "]", e);
-            listener.onFailure(
-                new IllegalStateException(
-                    String.format(
-                        Locale.ROOT,
-                        "failure when sending a join validation request from [%s] to [%s]",
-                        transportService.getLocalNode().descriptionWithoutAttributes(),
-                        discoveryNode.descriptionWithoutAttributes()
-                    ),
-                    e
-                )
-            );
-        }));
-        final var clusterState = clusterStateSupplier.get();
-        if (clusterState != null) {
-            assert clusterState.nodes().isLocalNodeElectedMaster();
-            transportService.sendRequest(
-                connection,
-                JOIN_VALIDATE_ACTION_NAME,
-                new ValidateJoinRequest(clusterState),
-                REQUEST_OPTIONS,
-                responseHandler
-            );
-        } else {
-            transportService.sendRequest(
-                connection,
-                JoinHelper.JOIN_PING_ACTION_NAME,
-                new JoinHelper.JoinPingRequest(),
-                REQUEST_OPTIONS,
-                responseHandler
-            );
+            listener.onFailure(new NodeClosedException(transportService.getLocalNode()));
         }
     }
 
@@ -341,7 +297,6 @@ private class JoinValidation extends ActionRunnable<Void> {
 
         @Override
         protected void doRun() {
-            assert connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) : discoveryNode.getVersion();
             // NB these things never run concurrently to each other, or to the cache cleaner (see IMPLEMENTATION NOTES above) so it is safe
             // to do these (non-atomic) things to the (unsynchronized) statesByVersion map.
             var transportVersion = connection.getTransportVersion();
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java
index d73143850b64f..124c17d705378 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.PrioritizedThrottledTaskRunner;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
@@ -271,9 +270,7 @@ public void onFailure(Exception e) {
                     @Override
                     public void onResponse(Releasable releasable) {
                         boolean success = false;
-                        final ThreadContext threadContext = transportService.getThreadPool().getThreadContext();
-                        try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
-                            threadContext.markAsSystemContext();
+                        try (var ignored = transportService.getThreadPool().getThreadContext().newEmptySystemContext()) {
                             client.execute(
                                 TransportNodesHotThreadsAction.TYPE,
                                 new NodesHotThreadsRequest(
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java
index 1d99f28e62582..c81e4877196b3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.cluster.coordination;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -29,19 +28,12 @@ public class ValidateJoinRequest extends TransportRequest {
 
     public ValidateJoinRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0)) {
-            // recent versions send a BytesTransportRequest containing a compressed representation of the state
-            final var bytes = in.readReleasableBytesReference();
-            final var version = in.getTransportVersion();
-            final var namedWriteableRegistry = in.namedWriteableRegistry();
-            this.stateSupplier = () -> readCompressed(version, bytes, namedWriteableRegistry);
-            this.refCounted = bytes;
-        } else {
-            // older versions just contain the bare state
-            final var state = ClusterState.readFrom(in, null);
-            this.stateSupplier = () -> state;
-            this.refCounted = null;
-        }
+        // recent versions send a BytesTransportRequest containing a compressed representation of the state
+        final var bytes = in.readReleasableBytesReference();
+        final var version = in.getTransportVersion();
+        final var namedWriteableRegistry = in.namedWriteableRegistry();
+        this.stateSupplier = () -> readCompressed(version, bytes, namedWriteableRegistry);
+        this.refCounted = bytes;
     }
 
     private static ClusterState readCompressed(
@@ -68,7 +60,6 @@ public ValidateJoinRequest(ClusterState state) {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        assert out.getTransportVersion().before(TransportVersions.V_8_3_0);
         super.writeTo(out);
         stateSupplier.get().writeTo(out);
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
index 6d1a874e1c72b..ae7cff6312155 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
@@ -28,6 +28,8 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -189,9 +191,14 @@ public List<String> getRequiredComponentTemplates() {
         if (ignoreMissingComponentTemplates == null) {
             return componentTemplates;
         }
-        return componentTemplates.stream()
-            .filter(componentTemplate -> ignoreMissingComponentTemplates.contains(componentTemplate) == false)
-            .toList();
+        // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+        List<String> required = new ArrayList<>(componentTemplates.size());
+        for (String template : componentTemplates) {
+            if (ignoreMissingComponentTemplates.contains(template) == false) {
+                required.add(template);
+            }
+        }
+        return Collections.unmodifiableList(required);
     }
 
     @Nullable
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
index dd4a52fd9beda..4dcc7c73c280e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -211,7 +211,7 @@ public static DataStream read(StreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(DataStream.ADDED_AUTO_SHARDING_EVENT_VERSION)) {
             backingIndicesBuilder.setAutoShardingEvent(in.readOptionalWriteable(DataStreamAutoShardingEvent::new));
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_FIELD_PARITY)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             failureIndicesBuilder.setRolloverOnWrite(in.readBoolean())
                 .setAutoShardingEvent(in.readOptionalWriteable(DataStreamAutoShardingEvent::new));
         }
@@ -1089,7 +1089,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(DataStream.ADDED_AUTO_SHARDING_EVENT_VERSION)) {
             out.writeOptionalWriteable(backingIndices.autoShardingEvent);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_FIELD_PARITY)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBoolean(failureIndices.rolloverOnWrite);
             out.writeOptionalWriteable(failureIndices.autoShardingEvent);
         }
@@ -1343,7 +1343,7 @@ public Index getWriteIndex(IndexRequest request, Metadata metadata) {
                         + "]"
                 )
                 .collect(Collectors.joining());
-            throw new IllegalArgumentException(
+            throw new TimestampError(
                 "the document timestamp ["
                     + timestampAsString
                     + "] is outside of ranges of currently writable indices ["
@@ -1405,10 +1405,10 @@ private static Instant getTimeStampFromRaw(Object rawTimestamp) {
             } else if (rawTimestamp instanceof String sTimestamp) {
                 return DateFormatters.from(TIMESTAMP_FORMATTER.parse(sTimestamp), TIMESTAMP_FORMATTER.locale()).toInstant();
             } else {
-                throw new IllegalArgumentException("timestamp [" + rawTimestamp + "] type [" + rawTimestamp.getClass() + "] error");
+                throw new TimestampError("timestamp [" + rawTimestamp + "] type [" + rawTimestamp.getClass() + "] error");
             }
         } catch (Exception e) {
-            throw new IllegalArgumentException("Error get data stream timestamp field: " + e.getMessage(), e);
+            throw new TimestampError("Error get data stream timestamp field: " + e.getMessage(), e);
         }
     }
 
@@ -1432,7 +1432,7 @@ private static Instant getTimestampFromParser(BytesReference source, XContentTyp
                 );
             };
         } catch (Exception e) {
-            throw new IllegalArgumentException("Error extracting data stream timestamp field: " + e.getMessage(), e);
+            throw new TimestampError("Error extracting data stream timestamp field: " + e.getMessage(), e);
         }
     }
 
@@ -1741,4 +1741,20 @@ public DataStream build() {
             );
         }
     }
+
+    /**
+     * This is a specialised error to capture that a document does not have a valid timestamp
+     * to index a document. It is mainly applicable for TSDS data streams because they need the timestamp
+     * to determine the write index.
+     */
+    public static class TimestampError extends IllegalArgumentException {
+
+        public TimestampError(String message, Exception cause) {
+            super(message, cause);
+        }
+
+        public TimestampError(String message) {
+            super(message);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java
index fd3fc1a732acb..7315e9f7a51d3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -19,6 +20,8 @@
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * A utility class that contains the mappings and settings logic for failure store indices that are a part of data streams.
@@ -26,12 +29,30 @@
 public class DataStreamFailureStoreDefinition {
 
     public static final String FAILURE_STORE_REFRESH_INTERVAL_SETTING_NAME = "data_streams.failure_store.refresh_interval";
+    public static final String INDEX_FAILURE_STORE_VERSION_SETTING_NAME = "index.failure_store.version";
     public static final Settings DATA_STREAM_FAILURE_STORE_SETTINGS;
+    // Only a subset of user configurable settings is applicable for a failure index. Here we have an
+    // allowlist that will filter all other settings out.
+    public static final Set<String> SUPPORTED_USER_SETTINGS = Set.of(
+        DataTier.TIER_PREFERENCE,
+        IndexMetadata.SETTING_INDEX_HIDDEN,
+        INDEX_FAILURE_STORE_VERSION_SETTING_NAME,
+        IndexMetadata.SETTING_NUMBER_OF_SHARDS,
+        IndexMetadata.SETTING_NUMBER_OF_REPLICAS,
+        IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS,
+        IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(),
+        IndexMetadata.LIFECYCLE_NAME
+    );
+    public static final Set<String> SUPPORTED_USER_SETTINGS_PREFIXES = Set.of(
+        IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".",
+        IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + ".",
+        IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "."
+    );
     public static final CompressedXContent DATA_STREAM_FAILURE_STORE_MAPPING;
 
     public static final int FAILURE_STORE_DEFINITION_VERSION = 1;
     public static final Setting<Integer> FAILURE_STORE_DEFINITION_VERSION_SETTING = Setting.intSetting(
-        "index.failure_store.version",
+        INDEX_FAILURE_STORE_VERSION_SETTING_NAME,
         0,
         Setting.Property.IndexScope
     );
@@ -40,11 +61,6 @@ public class DataStreamFailureStoreDefinition {
         DATA_STREAM_FAILURE_STORE_SETTINGS = Settings.builder()
             // Always start with the hidden settings for a backing index.
             .put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
-            // Override any pipeline settings on the failure store to not use any
-            // specified by the data stream template. Default pipelines are very much
-            // meant for the backing indices only.
-            .putNull(IndexSettings.DEFAULT_PIPELINE.getKey())
-            .putNull(IndexSettings.FINAL_PIPELINE.getKey())
             .put(FAILURE_STORE_DEFINITION_VERSION_SETTING.getKey(), FAILURE_STORE_DEFINITION_VERSION)
             .build();
 
@@ -199,4 +215,23 @@ public static Settings.Builder applyFailureStoreSettings(Settings nodeSettings,
         }
         return builder;
     }
+
+    /**
+     * Removes the unsupported by the failure store settings from the settings provided.
+     * ATTENTION: This method should be applied BEFORE we set the necessary settings for an index
+     * @param builder the settings builder that is going to be updated
+     * @return the original settings builder, with the unsupported settings removed.
+     */
+    public static Settings.Builder filterUserDefinedSettings(Settings.Builder builder) {
+        if (builder.keys().isEmpty() == false) {
+            Set<String> existingKeys = new HashSet<>(builder.keys());
+            for (String setting : existingKeys) {
+                if (SUPPORTED_USER_SETTINGS.contains(setting) == false
+                    && SUPPORTED_USER_SETTINGS_PREFIXES.stream().anyMatch(setting::startsWith) == false) {
+                    builder.remove(setting);
+                }
+            }
+        }
+        return builder;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
index fb8559b19d81d..fe72a59565cf6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -22,7 +21,6 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.Processors;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -38,7 +36,6 @@
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.function.Predicate;
-import java.util.regex.Pattern;
 
 import static java.lang.String.format;
 import static org.elasticsearch.node.Node.NODE_EXTERNAL_ID_SETTING;
@@ -58,8 +55,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
     private static final ParseField PROCESSORS_RANGE_FIELD = new ParseField("processors_range");
     private static final ParseField MEMORY_FIELD = new ParseField("memory");
     private static final ParseField STORAGE_FIELD = new ParseField("storage");
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // Remove deprecated field
-    private static final ParseField VERSION_FIELD = new ParseField("node_version");
 
     public static final ConstructingObjectParser<DesiredNode, Void> PARSER = new ConstructingObjectParser<>(
         "desired_node",
@@ -69,8 +64,7 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
             (Processors) args[1],
             (ProcessorsRange) args[2],
             (ByteSizeValue) args[3],
-            (ByteSizeValue) args[4],
-            (String) args[5]
+            (ByteSizeValue) args[4]
         )
     );
 
@@ -104,12 +98,6 @@ static <T> void configureParser(ConstructingObjectParser<T, Void> parser) {
             STORAGE_FIELD,
             ObjectParser.ValueType.STRING
         );
-        parser.declareField(
-            ConstructingObjectParser.optionalConstructorArg(),
-            (p, c) -> p.text(),
-            VERSION_FIELD,
-            ObjectParser.ValueType.STRING
-        );
     }
 
     private final Settings settings;
@@ -118,21 +106,9 @@ static <T> void configureParser(ConstructingObjectParser<T, Void> parser) {
     private final ByteSizeValue memory;
     private final ByteSizeValue storage;
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // Remove deprecated version field
-    private final String version;
     private final String externalId;
     private final Set<DiscoveryNodeRole> roles;
 
-    @Deprecated
-    public DesiredNode(Settings settings, ProcessorsRange processorsRange, ByteSizeValue memory, ByteSizeValue storage, String version) {
-        this(settings, null, processorsRange, memory, storage, version);
-    }
-
-    @Deprecated
-    public DesiredNode(Settings settings, double processors, ByteSizeValue memory, ByteSizeValue storage, String version) {
-        this(settings, Processors.of(processors), null, memory, storage, version);
-    }
-
     public DesiredNode(Settings settings, ProcessorsRange processorsRange, ByteSizeValue memory, ByteSizeValue storage) {
         this(settings, null, processorsRange, memory, storage);
     }
@@ -142,17 +118,6 @@ public DesiredNode(Settings settings, double processors, ByteSizeValue memory, B
     }
 
     DesiredNode(Settings settings, Processors processors, ProcessorsRange processorsRange, ByteSizeValue memory, ByteSizeValue storage) {
-        this(settings, processors, processorsRange, memory, storage, null);
-    }
-
-    DesiredNode(
-        Settings settings,
-        Processors processors,
-        ProcessorsRange processorsRange,
-        ByteSizeValue memory,
-        ByteSizeValue storage,
-        @Deprecated String version
-    ) {
         assert settings != null;
         assert memory != null;
         assert storage != null;
@@ -186,7 +151,6 @@ public DesiredNode(Settings settings, double processors, ByteSizeValue memory, B
         this.processorsRange = processorsRange;
         this.memory = memory;
         this.storage = storage;
-        this.version = version;
         this.externalId = NODE_EXTERNAL_ID_SETTING.get(settings);
         this.roles = Collections.unmodifiableSortedSet(new TreeSet<>(DiscoveryNode.getRolesFromSettings(settings)));
     }
@@ -210,19 +174,7 @@ public static DesiredNode readFrom(StreamInput in) throws IOException {
         } else {
             version = Version.readVersion(in).toString();
         }
-        return new DesiredNode(settings, processors, processorsRange, memory, storage, version);
-    }
-
-    private static final Pattern SEMANTIC_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)\\D?.*");
-
-    private static Version parseLegacyVersion(String version) {
-        if (version != null) {
-            var semanticVersionMatcher = SEMANTIC_VERSION_PATTERN.matcher(version);
-            if (semanticVersionMatcher.matches()) {
-                return Version.fromString(semanticVersionMatcher.group(1));
-            }
-        }
-        return null;
+        return new DesiredNode(settings, processors, processorsRange, memory, storage);
     }
 
     @Override
@@ -239,15 +191,9 @@ public void writeTo(StreamOutput out) throws IOException {
         memory.writeTo(out);
         storage.writeTo(out);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
-            out.writeOptionalString(version);
+            out.writeOptionalString(null);
         } else {
-            Version parsedVersion = parseLegacyVersion(version);
-            if (version == null) {
-                // Some node is from before we made the version field not required. If so, fill in with the current node version.
-                Version.writeVersion(Version.CURRENT, out);
-            } else {
-                Version.writeVersion(parsedVersion, out);
-            }
+            Version.writeVersion(Version.CURRENT, out);
         }
     }
 
@@ -275,14 +221,6 @@ public void toInnerXContent(XContentBuilder builder, Params params) throws IOExc
         }
         builder.field(MEMORY_FIELD.getPreferredName(), memory);
         builder.field(STORAGE_FIELD.getPreferredName(), storage);
-        addDeprecatedVersionField(builder);
-    }
-
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // Remove deprecated field from response
-    private void addDeprecatedVersionField(XContentBuilder builder) throws IOException {
-        if (version != null) {
-            builder.field(VERSION_FIELD.getPreferredName(), version);
-        }
     }
 
     public boolean hasMasterRole() {
@@ -366,7 +304,6 @@ private boolean equalsWithoutProcessorsSpecification(DesiredNode that) {
         return Objects.equals(settings, that.settings)
             && Objects.equals(memory, that.memory)
             && Objects.equals(storage, that.storage)
-            && Objects.equals(version, that.version)
             && Objects.equals(externalId, that.externalId)
             && Objects.equals(roles, that.roles);
     }
@@ -379,7 +316,7 @@ public boolean equalsWithProcessorsCloseTo(DesiredNode that) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(settings, processors, processorsRange, memory, storage, version, externalId, roles);
+        return Objects.hash(settings, processors, processorsRange, memory, storage, externalId, roles);
     }
 
     @Override
@@ -408,10 +345,6 @@ public String toString() {
             + '}';
     }
 
-    public boolean hasVersion() {
-        return Strings.isNullOrBlank(version) == false;
-    }
-
     public record ProcessorsRange(Processors min, @Nullable Processors max) implements Writeable, ToXContentObject {
 
         private static final ParseField MIN_FIELD = new ParseField("min");
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java
index 7b89406be9aa0..606309adf205c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java
@@ -44,13 +44,12 @@ public record DesiredNodeWithStatus(DesiredNode desiredNode, Status status)
                 (Processors) args[1],
                 (DesiredNode.ProcessorsRange) args[2],
                 (ByteSizeValue) args[3],
-                (ByteSizeValue) args[4],
-                (String) args[5]
+                (ByteSizeValue) args[4]
             ),
             // An unknown status is expected during upgrades to versions >= STATUS_TRACKING_SUPPORT_VERSION
             // the desired node status would be populated when a node in the newer version is elected as
             // master, the desired nodes status update happens in NodeJoinExecutor.
-            args[6] == null ? Status.PENDING : (Status) args[6]
+            args[5] == null ? Status.PENDING : (Status) args[5]
         )
     );
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 783145d3618f1..62867b4260bfd 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xcontent.ContextParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -128,8 +128,8 @@ public boolean containsIndex(final Index index) {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
-        return ChunkedToXContentHelper.array(TOMBSTONES_FIELD.getPreferredName(), tombstones.iterator());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContent.builder(params).array(TOMBSTONES_FIELD.getPreferredName(), tombstones.iterator());
     }
 
     public static IndexGraveyard fromXContent(final XContentParser parser) throws IOException {
@@ -434,7 +434,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
             builder.startObject();
             builder.field(INDEX_KEY);
             index.toXContent(builder, params);
-            builder.timeField(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis);
+            builder.timestampFieldsFromUnixEpochMillis(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis);
             return builder.endObject();
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 9760d84c67c5b..6456240c2317e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
@@ -945,9 +946,9 @@ public IndexMetadata withTimestampRanges(
         if (timestampRange.equals(this.timestampRange) && eventIngestedRange.equals(this.eventIngestedRange)) {
             return this;
         }
+        @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) // remove this check when 8.15 is no longer communicable
         IndexLongFieldRange allowedEventIngestedRange = eventIngestedRange;
-        // remove this check when the EVENT_INGESTED_RANGE_IN_CLUSTER_STATE version is removed
-        if (minClusterTransportVersion.before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+        if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) {
             allowedEventIngestedRange = IndexLongFieldRange.UNKNOWN;
         }
         return new IndexMetadata(
@@ -1644,7 +1645,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
                 DiffableUtils.getStringKeySerializer(),
                 ROLLOVER_INFO_DIFF_VALUE_READER
             );
-            if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 mappingsUpdatedVersion = IndexVersion.readVersion(in);
             } else {
                 mappingsUpdatedVersion = IndexVersions.ZERO;
@@ -1664,7 +1665,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
                 indexWriteLoadForecast = null;
                 shardSizeInBytesForecast = null;
             }
-            if (in.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 eventIngestedRange = IndexLongFieldRange.readFrom(in);
             } else {
                 eventIngestedRange = IndexLongFieldRange.UNKNOWN;
@@ -1698,7 +1699,7 @@ public void writeTo(StreamOutput out) throws IOException {
             customData.writeTo(out);
             inSyncAllocationIds.writeTo(out);
             rolloverInfos.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 IndexVersion.writeVersion(mappingsUpdatedVersion, out);
             }
             if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
@@ -1710,7 +1711,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeOptionalDouble(indexWriteLoadForecast);
                 out.writeOptionalLong(shardSizeInBytesForecast);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 eventIngestedRange.writeTo(out);
             } else {
                 assert eventIngestedRange == IndexLongFieldRange.UNKNOWN
@@ -1809,7 +1810,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
         for (int i = 0; i < rolloverAliasesSize; i++) {
             builder.putRolloverInfo(new RolloverInfo(in));
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             builder.mappingsUpdatedVersion(IndexVersion.readVersion(in));
         }
         if (in.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
@@ -1822,7 +1823,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
             builder.indexWriteLoadForecast(in.readOptionalDouble());
             builder.shardSizeInBytesForecast(in.readOptionalLong());
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             builder.eventIngestedRange(IndexLongFieldRange.readFrom(in), in.getTransportVersion());
         } else {
             builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, in.getTransportVersion());
@@ -1867,7 +1868,7 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException
             (o, v) -> DiffableUtils.StringSetValueSerializer.getInstance().write(v, o)
         );
         out.writeCollection(rolloverInfos.values());
-        if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             IndexVersion.writeVersion(mappingsUpdatedVersion, out);
         }
         if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
@@ -1879,7 +1880,7 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException
             out.writeOptionalDouble(writeLoadForecast);
             out.writeOptionalLong(shardSizeInBytesForecast);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             eventIngestedRange.writeTo(out);
         } else {
             assert eventIngestedRange == IndexLongFieldRange.UNKNOWN
@@ -2205,8 +2206,7 @@ public Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange, Transp
                     + minClusterTransportVersion
                     + "; eventIngestedRange = "
                     + eventIngestedRange;
-            if (minClusterTransportVersion != null
-                && minClusterTransportVersion.before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+            if (minClusterTransportVersion != null && minClusterTransportVersion.before(TransportVersions.V_8_15_0)) {
                 this.eventIngestedRange = IndexLongFieldRange.UNKNOWN;
             } else {
                 this.eventIngestedRange = eventIngestedRange;
@@ -2265,7 +2265,7 @@ IndexMetadata build(boolean repair) {
                     "routing partition size ["
                         + routingPartitionSize
                         + "] should be a positive number"
-                        + " less than the number of shards ["
+                        + " less than the number of routing shards ["
                         + getRoutingNumShards()
                         + "] for ["
                         + index
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 2229166a2d779..39499253c8790 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -48,7 +48,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -253,7 +252,7 @@ protected static Collection<String> resolveExpressions(Context context, String..
             } else {
                 return ExplicitResourceNameFilter.filterUnavailable(
                     context,
-                    DateMathExpressionResolver.resolve(context, List.of(expressions))
+                    DateMathExpressionResolver.resolve(context, Arrays.asList(expressions))
                 );
             }
         } else {
@@ -264,7 +263,10 @@ protected static Collection<String> resolveExpressions(Context context, String..
             } else {
                 return WildcardExpressionResolver.resolve(
                     context,
-                    ExplicitResourceNameFilter.filterUnavailable(context, DateMathExpressionResolver.resolve(context, List.of(expressions)))
+                    ExplicitResourceNameFilter.filterUnavailable(
+                        context,
+                        DateMathExpressionResolver.resolve(context, Arrays.asList(expressions))
+                    )
                 );
             }
         }
@@ -1294,34 +1296,51 @@ private static boolean shouldIncludeIfAlias(IndexAbstraction ia, IndexNameExpres
          * </ol>
          */
         public static Collection<String> resolve(Context context, List<String> expressions) {
-            ExpressionList expressionList = new ExpressionList(context, expressions);
             // fast exit if there are no wildcards to evaluate
-            if (expressionList.hasWildcard() == false) {
+            if (context.getOptions().expandWildcardExpressions() == false) {
+                return expressions;
+            }
+            int firstWildcardIndex = 0;
+            for (; firstWildcardIndex < expressions.size(); firstWildcardIndex++) {
+                String expression = expressions.get(firstWildcardIndex);
+                if (isWildcard(expression)) {
+                    break;
+                }
+            }
+            if (firstWildcardIndex == expressions.size()) {
                 return expressions;
             }
             Set<String> result = new HashSet<>();
-            for (ExpressionList.Expression expression : expressionList) {
-                if (expression.isWildcard()) {
-                    Stream<IndexAbstraction> matchingResources = matchResourcesToWildcard(context, expression.get());
+            for (int i = 0; i < firstWildcardIndex; i++) {
+                result.add(expressions.get(i));
+            }
+            AtomicBoolean emptyWildcardExpansion = context.getOptions().allowNoIndices() ? null : new AtomicBoolean();
+            for (int i = firstWildcardIndex; i < expressions.size(); i++) {
+                String expression = expressions.get(i);
+                boolean isExclusion = i > firstWildcardIndex && expression.charAt(0) == '-';
+                if (i == firstWildcardIndex || isWildcard(expression)) {
+                    Stream<IndexAbstraction> matchingResources = matchResourcesToWildcard(
+                        context,
+                        isExclusion ? expression.substring(1) : expression
+                    );
                     Stream<String> matchingOpenClosedNames = expandToOpenClosed(context, matchingResources);
-                    AtomicBoolean emptyWildcardExpansion = new AtomicBoolean(false);
-                    if (context.getOptions().allowNoIndices() == false) {
+                    if (emptyWildcardExpansion != null) {
                         emptyWildcardExpansion.set(true);
                         matchingOpenClosedNames = matchingOpenClosedNames.peek(x -> emptyWildcardExpansion.set(false));
                     }
-                    if (expression.isExclusion()) {
-                        matchingOpenClosedNames.forEachOrdered(result::remove);
+                    if (isExclusion) {
+                        matchingOpenClosedNames.forEach(result::remove);
                     } else {
-                        matchingOpenClosedNames.forEachOrdered(result::add);
+                        matchingOpenClosedNames.forEach(result::add);
                     }
-                    if (emptyWildcardExpansion.get()) {
-                        throw notFoundException(expression.get());
+                    if (emptyWildcardExpansion != null && emptyWildcardExpansion.get()) {
+                        throw notFoundException(expression);
                     }
                 } else {
-                    if (expression.isExclusion()) {
-                        result.remove(expression.get());
+                    if (isExclusion) {
+                        result.remove(expression.substring(1));
                     } else {
-                        result.add(expression.get());
+                        result.add(expression);
                     }
                 }
             }
@@ -1507,27 +1526,35 @@ private DateMathExpressionResolver() {
             // utility class
         }
 
+        /**
+         * Resolves date math expressions. If this is a noop the given {@code expressions} list is returned without copying.
+         * As a result callers of this method should not mutate the returned list. Mutating it may come with unexpected side effects.
+         */
         public static List<String> resolve(Context context, List<String> expressions) {
-            List<String> result = new ArrayList<>(expressions.size());
-            for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) {
-                result.add(resolveExpression(expression, context::getStartTime));
+            boolean wildcardSeen = false;
+            final boolean expandWildcards = context.getOptions().expandWildcardExpressions();
+            String[] result = null;
+            for (int i = 0, n = expressions.size(); i < n; i++) {
+                String expression = expressions.get(i);
+                // accepts date-math exclusions that are of the form "-<...{}>",f i.e. the "-" is outside the "<>" date-math template
+                boolean isExclusion = wildcardSeen && expression.startsWith("-");
+                wildcardSeen = wildcardSeen || (expandWildcards && isWildcard(expression));
+                String toResolve = isExclusion ? expression.substring(1) : expression;
+                String resolved = resolveExpression(toResolve, context::getStartTime);
+                if (toResolve != resolved) {
+                    if (result == null) {
+                        result = expressions.toArray(Strings.EMPTY_ARRAY);
+                    }
+                    result[i] = isExclusion ? "-" + resolved : resolved;
+                }
             }
-            return result;
+            return result == null ? expressions : Arrays.asList(result);
         }
 
         static String resolveExpression(String expression) {
             return resolveExpression(expression, System::currentTimeMillis);
         }
 
-        static String resolveExpression(ExpressionList.Expression expression, LongSupplier getTime) {
-            if (expression.isExclusion()) {
-                // accepts date-math exclusions that are of the form "-<...{}>", i.e. the "-" is outside the "<>" date-math template
-                return "-" + resolveExpression(expression.get(), getTime);
-            } else {
-                return resolveExpression(expression.get(), getTime);
-            }
-        }
-
         static String resolveExpression(String expression, LongSupplier getTime) {
             if (expression.startsWith(EXPRESSION_LEFT_BOUND) == false || expression.endsWith(EXPRESSION_RIGHT_BOUND) == false) {
                 return expression;
@@ -1689,14 +1716,35 @@ private ExplicitResourceNameFilter() {
          */
         public static List<String> filterUnavailable(Context context, List<String> expressions) {
             ensureRemoteIndicesRequireIgnoreUnavailable(context.getOptions(), expressions);
-            List<String> result = new ArrayList<>(expressions.size());
-            for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) {
-                validateAliasOrIndex(expression);
-                if (expression.isWildcard() || expression.isExclusion() || ensureAliasOrIndexExists(context, expression.get())) {
-                    result.add(expression.expression());
+            final boolean expandWildcards = context.getOptions().expandWildcardExpressions();
+            boolean wildcardSeen = false;
+            List<String> result = null;
+            for (int i = 0; i < expressions.size(); i++) {
+                String expression = expressions.get(i);
+                if (Strings.isEmpty(expression)) {
+                    throw notFoundException(expression);
+                }
+                // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API
+                // does not exist and the path is interpreted as an expression. If the expression begins with an underscore,
+                // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown
+                // if the expression can't be found.
+                if (expression.charAt(0) == '_') {
+                    throw new InvalidIndexNameException(expression, "must not start with '_'.");
+                }
+                final boolean isWildcard = expandWildcards && isWildcard(expression);
+                if (isWildcard || (wildcardSeen && expression.charAt(0) == '-') || ensureAliasOrIndexExists(context, expression)) {
+                    if (result != null) {
+                        result.add(expression);
+                    }
+                } else {
+                    if (result == null) {
+                        result = new ArrayList<>(expressions.size() - 1);
+                        result.addAll(expressions.subList(0, i));
+                    }
                 }
+                wildcardSeen |= isWildcard;
             }
-            return result;
+            return result == null ? expressions : result;
         }
 
         /**
@@ -1736,19 +1784,6 @@ private static boolean ensureAliasOrIndexExists(Context context, String name) {
             return true;
         }
 
-        private static void validateAliasOrIndex(ExpressionList.Expression expression) {
-            if (Strings.isEmpty(expression.expression())) {
-                throw notFoundException(expression.expression());
-            }
-            // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API
-            // does not exist and the path is interpreted as an expression. If the expression begins with an underscore,
-            // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown
-            // if the expression can't be found.
-            if (expression.expression().charAt(0) == '_') {
-                throw new InvalidIndexNameException(expression.expression(), "must not start with '_'.");
-            }
-        }
-
         private static void ensureRemoteIndicesRequireIgnoreUnavailable(IndicesOptions options, List<String> indexExpressions) {
             if (options.ignoreUnavailable()) {
                 return;
@@ -1773,57 +1808,6 @@ private static void failOnRemoteIndicesNotIgnoringUnavailable(List<String> index
         }
     }
 
-    /**
-     * Used to iterate expression lists and work out which expression item is a wildcard or an exclusion.
-     */
-    public static final class ExpressionList implements Iterable<ExpressionList.Expression> {
-        private final List<Expression> expressionsList;
-        private final boolean hasWildcard;
-
-        public record Expression(String expression, boolean isWildcard, boolean isExclusion) {
-            public String get() {
-                if (isExclusion()) {
-                    // drop the leading "-" if exclusion because it is easier for callers to handle it like this
-                    return expression().substring(1);
-                } else {
-                    return expression();
-                }
-            }
-        }
-
-        /**
-         * Creates the expression iterable that can be used to easily check which expression item is a wildcard or an exclusion (or both).
-         * The {@param context} is used to check if wildcards ought to be considered or not.
-         */
-        public ExpressionList(Context context, List<String> expressionStrings) {
-            List<Expression> expressionsList = new ArrayList<>(expressionStrings.size());
-            boolean wildcardSeen = false;
-            for (String expressionString : expressionStrings) {
-                boolean isExclusion = expressionString.startsWith("-") && wildcardSeen;
-                if (context.getOptions().expandWildcardExpressions() && isWildcard(expressionString)) {
-                    wildcardSeen = true;
-                    expressionsList.add(new Expression(expressionString, true, isExclusion));
-                } else {
-                    expressionsList.add(new Expression(expressionString, false, isExclusion));
-                }
-            }
-            this.expressionsList = expressionsList;
-            this.hasWildcard = wildcardSeen;
-        }
-
-        /**
-         * Returns {@code true} if the expression contains any wildcard and the options allow wildcard expansion
-         */
-        public boolean hasWildcard() {
-            return this.hasWildcard;
-        }
-
-        @Override
-        public Iterator<ExpressionList.Expression> iterator() {
-            return expressionsList.iterator();
-        }
-    }
-
     /**
      * This is a context for the DateMathExpressionResolver which does not require {@code IndicesOptions} or {@code ClusterState}
      * since it uses only the start time to resolve expressions.
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java
index 6ddcd6a45e4b6..1379489182b53 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java
@@ -35,9 +35,6 @@
 import java.util.Objects;
 import java.util.Set;
 
-import static org.elasticsearch.core.RestApiVersion.V_8;
-import static org.elasticsearch.core.RestApiVersion.onOrAfter;
-
 public class IndexTemplateMetadata implements SimpleDiffable<IndexTemplateMetadata> {
 
     private final String name;
@@ -379,9 +376,7 @@ private static void toInnerXContent(
             indexTemplateMetadata.settings().toXContent(builder, params);
             builder.endObject();
 
-            if (builder.getRestApiVersion().matches(onOrAfter(V_8))) {
-                includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false);
-            }
+            includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false);
 
             CompressedXContent m = indexTemplateMetadata.mappings();
             if (m != null) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
index 0756080c16d00..b7777eca86179 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
@@ -1316,23 +1316,6 @@ public Map<String, ComposableIndexTemplate> templatesV2() {
             .orElse(Collections.emptyMap());
     }
 
-    // TODO: remove this method:
-    public boolean isTimeSeriesTemplate(ComposableIndexTemplate indexTemplate) {
-        var indexModeFromTemplate = retrieveIndexModeFromTemplate(indexTemplate);
-        if (indexModeFromTemplate == IndexMode.TIME_SERIES) {
-            // No need to check for the existence of index.routing_path here, because index.mode=time_series can't be specified without it.
-            // Setting validation takes care of this.
-            // Also no need to validate that the fields defined in index.routing_path are keyword fields with time_series_dimension
-            // attribute enabled. This is validated elsewhere (DocumentMapper).
-            return true;
-        }
-
-        // in a followup change: check the existence of keyword fields of type keyword and time_series_dimension attribute enabled in
-        // the template. In this case the index.routing_path setting can be generated from the mapping.
-
-        return false;
-    }
-
     public IndexMode retrieveIndexModeFromTemplate(ComposableIndexTemplate indexTemplate) {
         if (indexTemplate.getDataStreamTemplate() == null) {
             return null;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
index 2df9cf706d892..5dbf4da6f376f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
@@ -425,7 +425,8 @@ public static ClusterState createFailureStoreIndex(
             .nameResolvedInstant(nameResolvedInstant)
             .performReroute(false)
             .setMatchingTemplate(template)
-            .settings(indexSettings);
+            .settings(indexSettings)
+            .isFailureIndex(true);
 
         try {
             currentState = metadataCreateIndexService.applyCreateIndexRequest(
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
index 1cebbabde0769..ed029db54bf06 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
@@ -308,7 +308,12 @@ private void onlyCreateIndex(
         final CreateIndexClusterStateUpdateRequest request,
         final ActionListener<AcknowledgedResponse> listener
     ) {
-        normalizeRequestSetting(request);
+        try {
+            normalizeRequestSetting(request);
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
 
         var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext());
         submitUnbatchedTask(
@@ -982,20 +987,22 @@ static Settings aggregateIndexSettings(
         if (sourceMetadata == null) {
             final Settings templateAndRequestSettings = Settings.builder().put(combinedTemplateSettings).put(request.settings()).build();
 
-            final boolean timeSeriesTemplate = Optional.of(request)
+            final IndexMode templateIndexMode = Optional.of(request)
+                .filter(r -> r.isFailureIndex() == false)
                 .map(CreateIndexClusterStateUpdateRequest::matchingTemplate)
-                .map(metadata::isTimeSeriesTemplate)
-                .orElse(false);
+                .map(metadata::retrieveIndexModeFromTemplate)
+                .orElse(null);
 
             // Loop through all the explicit index setting providers, adding them to the
             // additionalIndexSettings map
             final Settings.Builder additionalIndexSettings = Settings.builder();
             final var resolvedAt = Instant.ofEpochMilli(request.getNameResolvedAt());
+            Set<String> overrulingSettings = new HashSet<>();
             for (IndexSettingProvider provider : indexSettingProviders) {
                 var newAdditionalSettings = provider.getAdditionalIndexSettings(
                     request.index(),
                     request.dataStreamName(),
-                    timeSeriesTemplate,
+                    templateIndexMode,
                     currentState.getMetadata(),
                     resolvedAt,
                     templateAndRequestSettings,
@@ -1003,46 +1010,57 @@ static Settings aggregateIndexSettings(
                 );
                 validateAdditionalSettings(provider, newAdditionalSettings, additionalIndexSettings);
                 additionalIndexSettings.put(newAdditionalSettings);
+                if (provider.overrulesTemplateAndRequestSettings()) {
+                    overrulingSettings.addAll(newAdditionalSettings.keySet());
+                }
             }
 
-            // For all the explicit settings, we go through the template and request level settings
-            // and see if either a template or the request has "cancelled out" an explicit default
-            // setting. For example, if a plugin had as an explicit setting:
-            // "index.mysetting": "blah
-            // And either a template or create index request had:
-            // "index.mysetting": null
-            // We want to remove the explicit setting not only from the explicitly set settings, but
-            // also from the template and request settings, so that from the newly create index's
-            // perspective it is as though the setting has not been set at all (using the default
-            // value).
             for (String explicitSetting : additionalIndexSettings.keys()) {
-                if (templateSettings.keys().contains(explicitSetting) && templateSettings.get(explicitSetting) == null) {
-                    logger.debug(
-                        "removing default [{}] setting as it in set to null in a template for [{}] creation",
-                        explicitSetting,
-                        request.index()
-                    );
-                    additionalIndexSettings.remove(explicitSetting);
+                if (overrulingSettings.contains(explicitSetting)) {
+                    // Remove any conflicting template and request settings to use the provided values.
                     templateSettings.remove(explicitSetting);
-                }
-                if (requestSettings.keys().contains(explicitSetting) && requestSettings.get(explicitSetting) == null) {
-                    logger.debug(
-                        "removing default [{}] setting as it in set to null in the request for [{}] creation",
-                        explicitSetting,
-                        request.index()
-                    );
-                    additionalIndexSettings.remove(explicitSetting);
                     requestSettings.remove(explicitSetting);
+                } else {
+                    // For all the explicit settings, we go through the template and request level settings
+                    // and see if either a template or the request has "cancelled out" an explicit default
+                    // setting. For example, if a plugin had as an explicit setting:
+                    // "index.mysetting": "blah
+                    // And either a template or create index request had:
+                    // "index.mysetting": null
+                    // We want to remove the explicit setting not only from the explicitly set settings, but
+                    // also from the template and request settings, so that from the newly create index's
+                    // perspective it is as though the setting has not been set at all (using the default
+                    // value).
+                    if (templateSettings.keys().contains(explicitSetting) && templateSettings.get(explicitSetting) == null) {
+                        logger.debug(
+                            "removing default [{}] setting as it is set to null in a template for [{}] creation",
+                            explicitSetting,
+                            request.index()
+                        );
+                        additionalIndexSettings.remove(explicitSetting);
+                        templateSettings.remove(explicitSetting);
+                    }
+                    if (requestSettings.keys().contains(explicitSetting) && requestSettings.get(explicitSetting) == null) {
+                        logger.debug(
+                            "removing default [{}] setting as it is set to null in the request for [{}] creation",
+                            explicitSetting,
+                            request.index()
+                        );
+                        additionalIndexSettings.remove(explicitSetting);
+                        requestSettings.remove(explicitSetting);
+                    }
                 }
             }
 
             // Finally, we actually add the explicit defaults prior to the template settings and the
             // request settings, so that the precedence goes:
-            // Explicit Defaults -> Template -> Request -> Necessary Settings (# of shards, uuid, etc)
+            // Explicit Defaults -> Template -> Request -> Filter out failure store settings -> Necessary Settings (# of shards, uuid, etc)
             indexSettingsBuilder.put(additionalIndexSettings.build());
             indexSettingsBuilder.put(templateSettings.build());
         }
-
+        if (request.isFailureIndex()) {
+            DataStreamFailureStoreDefinition.filterUserDefinedSettings(indexSettingsBuilder);
+        }
         // now, put the request settings, so they override templates
         indexSettingsBuilder.put(requestSettings.build());
 
@@ -1306,7 +1324,7 @@ static IndexMetadata buildIndexMetadata(
     ) {
         IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards);
         indexMetadataBuilder.system(isSystem);
-        if (minClusterTransportVersion.before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+        if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) {
             // promote to UNKNOWN for older versions since they don't know how to handle event.ingested in cluster state
             indexMetadataBuilder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, minClusterTransportVersion);
         }
@@ -1373,7 +1391,7 @@ private static void updateIndexMappingsAndBuildSortOrder(
         MapperService mapperService = indexService.mapperService();
         IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD;
         List<CompressedXContent> allMappings = new ArrayList<>();
-        final CompressedXContent defaultMapping = indexMode.getDefaultMapping();
+        final CompressedXContent defaultMapping = indexMode.getDefaultMapping(indexService.getIndexSettings());
         if (defaultMapping != null) {
             allMappings.add(defaultMapping);
         }
@@ -1586,6 +1604,15 @@ static IndexMetadata validateResize(
             // of if the source shards are divisible by the number of target shards
             IndexMetadata.getRoutingFactor(sourceMetadata.getNumberOfShards(), INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings));
         }
+        if (targetIndexSettings.hasValue(IndexSettings.MODE.getKey())) {
+            IndexMode oldMode = Objects.requireNonNullElse(sourceMetadata.getIndexMode(), IndexMode.STANDARD);
+            IndexMode newMode = IndexSettings.MODE.get(targetIndexSettings);
+            if (newMode != oldMode) {
+                throw new IllegalArgumentException(
+                    "can't change index.mode of index [" + sourceIndex + "] from [" + oldMode + "] to [" + newMode + "]"
+                );
+            }
+        }
         return sourceMetadata;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
index 8a46550f8a689..db3973c1a15a8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
@@ -45,6 +45,7 @@ public class MetadataDataStreamsService {
     private final DataStreamGlobalRetentionSettings globalRetentionSettings;
     private final MasterServiceTaskQueue<UpdateLifecycleTask> updateLifecycleTaskQueue;
     private final MasterServiceTaskQueue<SetRolloverOnWriteTask> setRolloverOnWriteTaskQueue;
+    private final MasterServiceTaskQueue<UpdateOptionsTask> updateOptionsTaskQueue;
 
     public MetadataDataStreamsService(
         ClusterService clusterService,
@@ -93,6 +94,20 @@ public Tuple<ClusterState, ClusterStateAckListener> executeTask(
             Priority.NORMAL,
             rolloverOnWriteExecutor
         );
+        ClusterStateTaskExecutor<UpdateOptionsTask> updateOptionsExecutor = new SimpleBatchedAckListenerTaskExecutor<>() {
+
+            @Override
+            public Tuple<ClusterState, ClusterStateAckListener> executeTask(
+                UpdateOptionsTask modifyOptionsTask,
+                ClusterState clusterState
+            ) {
+                return new Tuple<>(
+                    updateDataStreamOptions(clusterState, modifyOptionsTask.getDataStreamNames(), modifyOptionsTask.getOptions()),
+                    modifyOptionsTask
+                );
+            }
+        };
+        this.updateOptionsTaskQueue = clusterService.createTaskQueue("modify-data-stream-options", Priority.NORMAL, updateOptionsExecutor);
     }
 
     public void modifyDataStream(final ModifyDataStreamsAction.Request request, final ActionListener<AcknowledgedResponse> listener) {
@@ -147,6 +162,39 @@ public void removeLifecycle(
         );
     }
 
+    /**
+     * Submits the task to set the provided data stream options to the requested data streams.
+     */
+    public void setDataStreamOptions(
+        final List<String> dataStreamNames,
+        DataStreamOptions options,
+        TimeValue ackTimeout,
+        TimeValue masterTimeout,
+        final ActionListener<AcknowledgedResponse> listener
+    ) {
+        updateOptionsTaskQueue.submitTask(
+            "set-data-stream-options",
+            new UpdateOptionsTask(dataStreamNames, options, ackTimeout, listener),
+            masterTimeout
+        );
+    }
+
+    /**
+     * Submits the task to remove the data stream options from the requested data streams.
+     */
+    public void removeDataStreamOptions(
+        List<String> dataStreamNames,
+        TimeValue ackTimeout,
+        TimeValue masterTimeout,
+        ActionListener<AcknowledgedResponse> listener
+    ) {
+        updateOptionsTaskQueue.submitTask(
+            "delete-data-stream-options",
+            new UpdateOptionsTask(dataStreamNames, null, ackTimeout, listener),
+            masterTimeout
+        );
+    }
+
     @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here
     private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) {
         clusterService.submitUnbatchedStateUpdateTask(source, task);
@@ -228,6 +276,24 @@ ClusterState updateDataLifecycle(ClusterState currentState, List<String> dataStr
         return ClusterState.builder(currentState).metadata(builder.build()).build();
     }
 
+    /**
+     * Creates an updated cluster state in which the requested data streams have the data stream options provided.
+     * Visible for testing.
+     */
+    ClusterState updateDataStreamOptions(
+        ClusterState currentState,
+        List<String> dataStreamNames,
+        @Nullable DataStreamOptions dataStreamOptions
+    ) {
+        Metadata metadata = currentState.metadata();
+        Metadata.Builder builder = Metadata.builder(metadata);
+        for (var dataStreamName : dataStreamNames) {
+            var dataStream = validateDataStream(metadata, dataStreamName);
+            builder.put(dataStream.copy().setDataStreamOptions(dataStreamOptions).build());
+        }
+        return ClusterState.builder(currentState).metadata(builder.build()).build();
+    }
+
     /**
      * Creates an updated cluster state in which the requested data stream has the flag {@link DataStream#rolloverOnWrite()}
      * set to the value of the parameter rolloverOnWrite
@@ -372,6 +438,34 @@ public DataStreamLifecycle getDataLifecycle() {
         }
     }
 
+    /**
+     * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion.
+     */
+    static class UpdateOptionsTask extends AckedBatchedClusterStateUpdateTask {
+
+        private final List<String> dataStreamNames;
+        private final DataStreamOptions options;
+
+        UpdateOptionsTask(
+            List<String> dataStreamNames,
+            @Nullable DataStreamOptions options,
+            TimeValue ackTimeout,
+            ActionListener<AcknowledgedResponse> listener
+        ) {
+            super(ackTimeout, listener);
+            this.dataStreamNames = dataStreamNames;
+            this.options = options;
+        }
+
+        public List<String> getDataStreamNames() {
+            return dataStreamNames;
+        }
+
+        public DataStreamOptions getOptions() {
+            return options;
+        }
+    }
+
     /**
      * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion.
      */
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
index 57194ded9422e..d6ed28454df96 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -705,7 +705,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT
             var newAdditionalSettings = provider.getAdditionalIndexSettings(
                 "validate-index-name",
                 indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null,
-                indexTemplate.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(indexTemplate),
+                metadata.retrieveIndexModeFromTemplate(indexTemplate),
                 currentState.getMetadata(),
                 now,
                 combinedSettings,
@@ -1200,6 +1200,42 @@ static ClusterState innerPutTemplate(
         return ClusterState.builder(currentState).metadata(builder).build();
     }
 
+    /**
+     * A private, local alternative to elements.stream().anyMatch(predicate) for micro-optimization reasons.
+     */
+    private static <T> boolean anyMatch(final List<T> elements, final Predicate<T> predicate) {
+        for (T e : elements) {
+            if (predicate.test(e)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * A private, local alternative to elements.stream().noneMatch(predicate) for micro-optimization reasons.
+     */
+    private static <T> boolean noneMatch(final List<T> elements, final Predicate<T> predicate) {
+        for (T e : elements) {
+            if (predicate.test(e)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * A private, local alternative to elements.stream().filter(predicate).findFirst() for micro-optimization reasons.
+     */
+    private static <T> Optional<T> findFirst(final List<T> elements, final Predicate<T> predicate) {
+        for (T e : elements) {
+            if (predicate.test(e)) {
+                return Optional.of(e);
+            }
+        }
+        return Optional.empty();
+    }
+
     /**
      * Finds index templates whose index pattern matched with the given index name. In the case of
      * hidden indices, a template with a match all pattern or global template will not be returned.
@@ -1219,15 +1255,14 @@ public static List<IndexTemplateMetadata> findV1Templates(Metadata metadata, Str
         final List<IndexTemplateMetadata> matchedTemplates = new ArrayList<>();
         for (IndexTemplateMetadata template : metadata.templates().values()) {
             if (isHidden == null || isHidden == Boolean.FALSE) {
-                final boolean matched = template.patterns().stream().anyMatch(patternMatchPredicate);
-                if (matched) {
+                if (anyMatch(template.patterns(), patternMatchPredicate)) {
                     matchedTemplates.add(template);
                 }
             } else {
                 assert isHidden == Boolean.TRUE;
-                final boolean isNotMatchAllTemplate = template.patterns().stream().noneMatch(Regex::isMatchAllPattern);
+                final boolean isNotMatchAllTemplate = noneMatch(template.patterns(), Regex::isMatchAllPattern);
                 if (isNotMatchAllTemplate) {
-                    if (template.patterns().stream().anyMatch(patternMatchPredicate)) {
+                    if (anyMatch(template.patterns(), patternMatchPredicate)) {
                         matchedTemplates.add(template);
                     }
                 }
@@ -1238,19 +1273,21 @@ public static List<IndexTemplateMetadata> findV1Templates(Metadata metadata, Str
         // this is complex but if the index is not hidden in the create request but is hidden as the result of template application,
         // then we need to exclude global templates
         if (isHidden == null) {
-            final Optional<IndexTemplateMetadata> templateWithHiddenSetting = matchedTemplates.stream()
-                .filter(template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings()))
-                .findFirst();
+            final Optional<IndexTemplateMetadata> templateWithHiddenSetting = findFirst(
+                matchedTemplates,
+                template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings())
+            );
             if (templateWithHiddenSetting.isPresent()) {
                 final boolean templatedIsHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(templateWithHiddenSetting.get().settings());
                 if (templatedIsHidden) {
                     // remove the global templates
-                    matchedTemplates.removeIf(current -> current.patterns().stream().anyMatch(Regex::isMatchAllPattern));
+                    matchedTemplates.removeIf(current -> anyMatch(current.patterns(), Regex::isMatchAllPattern));
                 }
                 // validate that hidden didn't change
-                final Optional<IndexTemplateMetadata> templateWithHiddenSettingPostRemoval = matchedTemplates.stream()
-                    .filter(template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings()))
-                    .findFirst();
+                final Optional<IndexTemplateMetadata> templateWithHiddenSettingPostRemoval = findFirst(
+                    matchedTemplates,
+                    template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings())
+                );
                 if (templateWithHiddenSettingPostRemoval.isEmpty()
                     || templateWithHiddenSetting.get() != templateWithHiddenSettingPostRemoval.get()) {
                     throw new IllegalStateException(
@@ -1313,14 +1350,13 @@ static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(Met
              * built with a template that none of its indices match.
              */
             if (isHidden == false || template.getDataStreamTemplate() != null) {
-                final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate);
-                if (matched) {
+                if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
                     candidates.add(Tuple.tuple(name, template));
                 }
             } else {
-                final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern);
+                final boolean isNotMatchAllTemplate = noneMatch(template.indexPatterns(), Regex::isMatchAllPattern);
                 if (isNotMatchAllTemplate) {
-                    if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) {
+                    if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
                         candidates.add(Tuple.tuple(name, template));
                     }
                 }
@@ -1334,7 +1370,7 @@ static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(Met
     // Checks if a global template specifies the `index.hidden` setting. This check is important because a global
     // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation.
     private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, ComposableIndexTemplate template, String templateName) {
-        return template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)
+        return anyMatch(template.indexPatterns(), Regex::isMatchAllPattern)
             && IndexMetadata.INDEX_HIDDEN_SETTING.exists(resolveSettings(metadata, templateName));
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
index 4257543498c54..aa8b092ffcca0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
@@ -266,7 +266,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(NODE_ID_FIELD.getPreferredName(), nodeId);
             builder.field(TYPE_FIELD.getPreferredName(), type);
             builder.field(REASON_FIELD.getPreferredName(), reason);
-            builder.timeField(STARTED_AT_MILLIS_FIELD.getPreferredName(), STARTED_AT_READABLE_FIELD, startedAtMillis);
+            builder.timestampFieldsFromUnixEpochMillis(
+                STARTED_AT_MILLIS_FIELD.getPreferredName(),
+                STARTED_AT_READABLE_FIELD,
+                startedAtMillis
+            );
             builder.field(NODE_SEEN_FIELD.getPreferredName(), nodeSeen);
             if (allocationDelay != null) {
                 builder.field(ALLOCATION_DELAY_FIELD.getPreferredName(), allocationDelay.getStringRep());
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 9120e25b443d7..f7812d284f2af 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -32,7 +32,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO;
 import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
 
 public class OperationRouting {
@@ -306,14 +305,8 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null
     }
 
     public static boolean canSearchShard(ShardRouting shardRouting, ClusterState clusterState) {
-        // TODO: remove if and always return isSearchable (ES-9563)
         if (INDEX_FAST_REFRESH_SETTING.get(clusterState.metadata().index(shardRouting.index()).getSettings())) {
-            // Until all the cluster is upgraded, we send searches/gets to the primary (even if it has been upgraded) to execute locally.
-            if (clusterState.getMinTransportVersion().onOrAfter(FAST_REFRESH_RCO)) {
-                return shardRouting.isSearchable();
-            } else {
-                return shardRouting.isPromotableToPrimary();
-            }
+            return shardRouting.isPromotableToPrimary();
         } else {
             return shardRouting.isSearchable();
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
index 7bb97faa6b2d0..827cc378ef3a9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -140,7 +141,11 @@ public static Iterator<ToXContent> nodeDecisionsToXContentChunked(List<NodeAlloc
             return Collections.emptyIterator();
         }
 
-        return ChunkedToXContentHelper.array("node_allocation_decisions", nodeDecisions.iterator());
+        return Iterators.concat(
+            ChunkedToXContentHelper.startArray("node_allocation_decisions"),
+            nodeDecisions.iterator(),
+            ChunkedToXContentHelper.endArray()
+        );
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
index c0543a93ea9d0..a61199e26191c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContent;
@@ -296,7 +296,7 @@ && getNodeDecisions().stream()
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         checkDecisionState();
-        return Iterators.concat(Iterators.single((builder, p) -> {
+        return ChunkedToXContent.builder(params).append((builder, p) -> {
             builder.field("can_allocate", getAllocationDecision());
             builder.field("allocate_explanation", getExplanation());
             if (targetNode != null) {
@@ -320,7 +320,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 );
             }
             return builder;
-        }), nodeDecisionsToXContentChunked(nodeDecisions));
+        }).append(nodeDecisionsToXContentChunked(nodeDecisions));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java
index 3c559f9421a38..4c2f0cbaaf729 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
@@ -226,7 +227,7 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP
         public Settings getAdditionalIndexSettings(
             String indexName,
             @Nullable String dataStreamName,
-            boolean isTimeSeries,
+            IndexMode templateIndexMode,
             Metadata metadata,
             Instant resolvedAt,
             Settings indexTemplateAndCreateRequestSettings,
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
index d1d6a9761a758..57abbb8b8ed94 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.unit.RelativeByteSizeValue;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -156,19 +155,6 @@ public class DiskThresholdSettings implements Writeable {
     private volatile boolean enabled;
     private volatile TimeValue rerouteInterval;
 
-    static {
-        checkAutoReleaseIndexEnabled();
-    }
-
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // this check is unnecessary in v9
-    private static void checkAutoReleaseIndexEnabled() {
-        final String AUTO_RELEASE_INDEX_ENABLED_KEY = "es.disk.auto_release_flood_stage_block";
-        final String property = System.getProperty(AUTO_RELEASE_INDEX_ENABLED_KEY);
-        if (property != null) {
-            throw new IllegalArgumentException("system property [" + AUTO_RELEASE_INDEX_ENABLED_KEY + "] may not be set");
-        }
-    }
-
     public DiskThresholdSettings(Settings settings, ClusterSettings clusterSettings) {
         setLowWatermark(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.get(settings));
         setLowStageMaxHeadroom(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_MAX_HEADROOM_SETTING.get(settings));
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
index 891818b8e68f7..5dfac293de491 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
@@ -12,9 +12,9 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -260,7 +260,7 @@ public String getExplanation() {
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         checkDecisionState();
-        return Iterators.concat(Iterators.single((builder, p) -> {
+        return ChunkedToXContent.builder(params).append((builder, p) -> {
             if (targetNode != null) {
                 builder.startObject("target_node");
                 discoveryNodeToXContent(targetNode, true, builder);
@@ -289,7 +289,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 builder.field("move_explanation", getExplanation());
             }
             return builder;
-        }), nodeDecisionsToXContentChunked(nodeDecisions));
+        }).append(nodeDecisionsToXContentChunked(nodeDecisions));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java
new file mode 100644
index 0000000000000..436f1ac38c0c2
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.routing.allocation.allocator;
+
+import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
+import org.elasticsearch.telemetry.metric.LongWithAttributes;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.util.List;
+
+public class DesiredBalanceMetrics {
+
+    public static final DesiredBalanceMetrics NOOP = new DesiredBalanceMetrics(MeterRegistry.NOOP);
+    public static final String UNASSIGNED_SHARDS_METRIC_NAME = "es.allocator.desired_balance.shards.unassigned.current";
+    public static final String TOTAL_SHARDS_METRIC_NAME = "es.allocator.desired_balance.shards.current";
+    public static final String UNDESIRED_ALLOCATION_COUNT_METRIC_NAME = "es.allocator.desired_balance.allocations.undesired.current";
+    public static final String UNDESIRED_ALLOCATION_RATIO_METRIC_NAME = "es.allocator.desired_balance.allocations.undesired.ratio";
+
+    private volatile boolean nodeIsMaster = false;
+
+    /**
+     * Number of unassigned shards during last reconciliation
+     */
+    private volatile long unassignedShards;
+    /**
+     * Total number of assigned shards during last reconciliation
+     */
+    private volatile long totalAllocations;
+    /**
+     * Number of assigned shards during last reconciliation that are not allocated on desired node and need to be moved
+     */
+    private volatile long undesiredAllocations;
+
+    public void updateMetrics(long unassignedShards, long totalAllocations, long undesiredAllocations) {
+        this.unassignedShards = unassignedShards;
+        this.totalAllocations = totalAllocations;
+        this.undesiredAllocations = undesiredAllocations;
+    }
+
+    public DesiredBalanceMetrics(MeterRegistry meterRegistry) {
+        meterRegistry.registerLongsGauge(
+            UNASSIGNED_SHARDS_METRIC_NAME,
+            "Current number of unassigned shards",
+            "{shard}",
+            this::getUnassignedShardsMetrics
+        );
+        meterRegistry.registerLongsGauge(TOTAL_SHARDS_METRIC_NAME, "Total number of shards", "{shard}", this::getTotalAllocationsMetrics);
+        meterRegistry.registerLongsGauge(
+            UNDESIRED_ALLOCATION_COUNT_METRIC_NAME,
+            "Total number of shards allocated on undesired nodes excluding shutting down nodes",
+            "{shard}",
+            this::getUndesiredAllocationsMetrics
+        );
+        meterRegistry.registerDoublesGauge(
+            UNDESIRED_ALLOCATION_RATIO_METRIC_NAME,
+            "Ratio of undesired allocations to shard count excluding shutting down nodes",
+            "1",
+            this::getUndesiredAllocationsRatioMetrics
+        );
+    }
+
+    public void setNodeIsMaster(boolean nodeIsMaster) {
+        this.nodeIsMaster = nodeIsMaster;
+    }
+
+    public long unassignedShards() {
+        return unassignedShards;
+    }
+
+    public long totalAllocations() {
+        return totalAllocations;
+    }
+
+    public long undesiredAllocations() {
+        return undesiredAllocations;
+    }
+
+    private List<LongWithAttributes> getUnassignedShardsMetrics() {
+        return getIfPublishing(unassignedShards);
+    }
+
+    private List<LongWithAttributes> getTotalAllocationsMetrics() {
+        return getIfPublishing(totalAllocations);
+    }
+
+    private List<LongWithAttributes> getUndesiredAllocationsMetrics() {
+        return getIfPublishing(undesiredAllocations);
+    }
+
+    private List<LongWithAttributes> getIfPublishing(long value) {
+        if (nodeIsMaster) {
+            return List.of(new LongWithAttributes(value));
+        }
+        return List.of();
+    }
+
+    private List<DoubleWithAttributes> getUndesiredAllocationsRatioMetrics() {
+        if (nodeIsMaster) {
+            var total = totalAllocations;
+            var undesired = undesiredAllocations;
+            return List.of(new DoubleWithAttributes(total != 0 ? (double) undesired / total : 0.0));
+        }
+        return List.of();
+    }
+
+    public void zeroAllMetrics() {
+        unassignedShards = 0;
+        totalAllocations = 0;
+        undesiredAllocations = 0;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
index da52148919cdb..dced9214a3245 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
@@ -30,10 +30,6 @@
 import org.elasticsearch.gateway.PriorityComparator;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.telemetry.metric.DoubleGauge;
-import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
-import org.elasticsearch.telemetry.metric.LongGaugeMetric;
-import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Comparator;
@@ -73,23 +69,10 @@ public class DesiredBalanceReconciler {
     private double undesiredAllocationsLogThreshold;
     private final NodeAllocationOrdering allocationOrdering = new NodeAllocationOrdering();
     private final NodeAllocationOrdering moveOrdering = new NodeAllocationOrdering();
+    private final DesiredBalanceMetrics desiredBalanceMetrics;
 
-    // stats
-    /**
-     * Number of unassigned shards during last reconciliation
-     */
-    protected final LongGaugeMetric unassignedShards;
-    /**
-     * Total number of assigned shards during last reconciliation
-     */
-    protected final LongGaugeMetric totalAllocations;
-    /**
-     * Number of assigned shards during last reconciliation that are not allocated on desired node and need to be moved
-     */
-    protected final LongGaugeMetric undesiredAllocations;
-    private final DoubleGauge undesiredAllocationsRatio;
-
-    public DesiredBalanceReconciler(ClusterSettings clusterSettings, ThreadPool threadPool, MeterRegistry meterRegistry) {
+    public DesiredBalanceReconciler(ClusterSettings clusterSettings, ThreadPool threadPool, DesiredBalanceMetrics desiredBalanceMetrics) {
+        this.desiredBalanceMetrics = desiredBalanceMetrics;
         this.undesiredAllocationLogInterval = new FrequencyCappedAction(
             threadPool.relativeTimeInMillisSupplier(),
             TimeValue.timeValueMinutes(5)
@@ -99,35 +82,6 @@ public DesiredBalanceReconciler(ClusterSettings clusterSettings, ThreadPool thre
             UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING,
             value -> this.undesiredAllocationsLogThreshold = value
         );
-
-        unassignedShards = LongGaugeMetric.create(
-            meterRegistry,
-            "es.allocator.desired_balance.shards.unassigned.current",
-            "Current number of unassigned shards",
-            "{shard}"
-        );
-        totalAllocations = LongGaugeMetric.create(
-            meterRegistry,
-            "es.allocator.desired_balance.shards.current",
-            "Total number of shards",
-            "{shard}"
-        );
-        undesiredAllocations = LongGaugeMetric.create(
-            meterRegistry,
-            "es.allocator.desired_balance.allocations.undesired.current",
-            "Total number of shards allocated on undesired nodes excluding shutting down nodes",
-            "{shard}"
-        );
-        undesiredAllocationsRatio = meterRegistry.registerDoubleGauge(
-            "es.allocator.desired_balance.allocations.undesired.ratio",
-            "Ratio of undesired allocations to shard count excluding shutting down nodes",
-            "1",
-            () -> {
-                var total = totalAllocations.get();
-                var undesired = undesiredAllocations.get();
-                return new DoubleWithAttributes(total != 0 ? (double) undesired / total : 0.0);
-            }
-        );
     }
 
     public void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) {
@@ -578,9 +532,7 @@ private void balance() {
                 }
             }
 
-            DesiredBalanceReconciler.this.unassignedShards.set(unassignedShards);
-            DesiredBalanceReconciler.this.undesiredAllocations.set(undesiredAllocationsExcludingShuttingDownNodes);
-            DesiredBalanceReconciler.this.totalAllocations.set(totalAllocations);
+            desiredBalanceMetrics.updateMetrics(unassignedShards, totalAllocations, undesiredAllocationsExcludingShuttingDownNodes);
 
             maybeLogUndesiredAllocationsWarning(totalAllocations, undesiredAllocationsExcludingShuttingDownNodes, routingNodes.size());
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
index ba16915f2ad2b..4171100191211 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
@@ -64,6 +64,7 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator {
     private volatile DesiredBalance currentDesiredBalance = DesiredBalance.INITIAL;
     private volatile boolean resetCurrentDesiredBalance = false;
     private final Set<String> processedNodeShutdowns = new HashSet<>();
+    private final DesiredBalanceMetrics desiredBalanceMetrics;
 
     // stats
     protected final CounterMetric computationsSubmitted = new CounterMetric();
@@ -104,6 +105,7 @@ public DesiredBalanceShardsAllocator(
         DesiredBalanceReconcilerAction reconciler,
         TelemetryProvider telemetryProvider
     ) {
+        this.desiredBalanceMetrics = new DesiredBalanceMetrics(telemetryProvider.getMeterRegistry());
         this.delegateAllocator = delegateAllocator;
         this.threadPool = threadPool;
         this.reconciler = reconciler;
@@ -111,7 +113,7 @@ public DesiredBalanceShardsAllocator(
         this.desiredBalanceReconciler = new DesiredBalanceReconciler(
             clusterService.getClusterSettings(),
             threadPool,
-            telemetryProvider.getMeterRegistry()
+            desiredBalanceMetrics
         );
         this.desiredBalanceComputation = new ContinuousComputation<>(threadPool.generic()) {
 
@@ -168,6 +170,10 @@ public String toString() {
             if (event.localNodeMaster() == false) {
                 onNoLongerMaster();
             }
+            // Only update on change, to minimise volatile writes
+            if (event.localNodeMaster() != event.previousState().nodes().isLocalNodeElectedMaster()) {
+                desiredBalanceMetrics.setNodeIsMaster(event.localNodeMaster());
+            }
         });
     }
 
@@ -306,9 +312,9 @@ public DesiredBalanceStats getStats() {
             computedShardMovements.sum(),
             cumulativeComputationTime.count(),
             cumulativeReconciliationTime.count(),
-            desiredBalanceReconciler.unassignedShards.get(),
-            desiredBalanceReconciler.totalAllocations.get(),
-            desiredBalanceReconciler.undesiredAllocations.get()
+            desiredBalanceMetrics.unassignedShards(),
+            desiredBalanceMetrics.totalAllocations(),
+            desiredBalanceMetrics.undesiredAllocations()
         );
     }
 
@@ -318,10 +324,7 @@ private void onNoLongerMaster() {
             queue.completeAllAsNotMaster();
             pendingDesiredBalanceMoves.clear();
             desiredBalanceReconciler.clear();
-
-            desiredBalanceReconciler.unassignedShards.set(0);
-            desiredBalanceReconciler.totalAllocations.set(0);
-            desiredBalanceReconciler.undesiredAllocations.set(0);
+            desiredBalanceMetrics.zeroAllMetrics();
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
index a55522ff14c83..0ab842276efc4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.settings.Setting;
 
 /**
@@ -72,9 +73,11 @@ private static Decision debugDecision(Decision decision, UnassignedInfo info, in
             return Decision.single(
                 Decision.Type.NO,
                 NAME,
-                "shard has exceeded the maximum number of retries [%d] on failed allocation attempts - manually call [%s] to retry, [%s]",
+                "shard has exceeded the maximum number of retries [%d] on failed allocation attempts - "
+                    + "manually call [%s] to retry, and for more information, see [%s] [%s]",
                 maxRetries,
                 RETRY_FAILED_API,
+                ReferenceDocs.ALLOCATION_EXPLAIN_MAX_RETRY,
                 info.toString()
             );
         } else {
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index 3e1f436006aa7..c34d0d19988c8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -334,8 +334,7 @@ private void submitStateUpdateTask(
         final ThreadContext threadContext = threadPool.getThreadContext();
         final Supplier<ThreadContext.StoredContext> storedContextSupplier = threadContext.newRestorableContext(true);
 
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            threadContext.markAsSystemContext();
+        try (var ignore = threadContext.newEmptySystemContext()) {
             threadPoolExecutor.execute(
                 new UpdateTask(
                     priority,
diff --git a/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java b/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java
index 27d98f9ade203..cacad64ab1d4f 100644
--- a/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java
+++ b/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java
@@ -8,6 +8,7 @@
  */
 package org.elasticsearch.common;
 
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 
 import java.util.Collections;
@@ -81,6 +82,18 @@ public static BackoffPolicy exponentialBackoff(TimeValue initialDelay, int maxNu
         return new ExponentialBackoff((int) checkDelay(initialDelay).millis(), maxNumberOfRetries);
     }
 
+    /**
+     * Creates a new linear backoff policy with the provided configuration
+     *
+     * @param delayIncrement The amount by which to increment the delay on each retry
+     * @param maxNumberOfRetries The maximum number of retries
+     * @param maximumDelay The maximum delay
+     * @return A backoff policy with linear increase in wait time for retries.
+     */
+    public static BackoffPolicy linearBackoff(TimeValue delayIncrement, int maxNumberOfRetries, TimeValue maximumDelay) {
+        return new LinearBackoff(delayIncrement, maxNumberOfRetries, maximumDelay);
+    }
+
     /**
      * Wraps the backoff policy in one that calls a method every time a new backoff is taken from the policy.
      */
@@ -100,6 +113,11 @@ private static class NoBackoff extends BackoffPolicy {
         public Iterator<TimeValue> iterator() {
             return Collections.emptyIterator();
         }
+
+        @Override
+        public String toString() {
+            return "NoBackoff";
+        }
     }
 
     private static class ExponentialBackoff extends BackoffPolicy {
@@ -118,6 +136,11 @@ private ExponentialBackoff(int start, int numberOfElements) {
         public Iterator<TimeValue> iterator() {
             return new ExponentialBackoffIterator(start, numberOfElements);
         }
+
+        @Override
+        public String toString() {
+            return "ExponentialBackoff{start=" + start + ", numberOfElements=" + numberOfElements + '}';
+        }
     }
 
     private static class ExponentialBackoffIterator implements Iterator<TimeValue> {
@@ -163,6 +186,11 @@ private static final class ConstantBackoff extends BackoffPolicy {
         public Iterator<TimeValue> iterator() {
             return new ConstantBackoffIterator(delay, numberOfElements);
         }
+
+        @Override
+        public String toString() {
+            return "ConstantBackoff{delay=" + delay + ", numberOfElements=" + numberOfElements + '}';
+        }
     }
 
     private static final class ConstantBackoffIterator implements Iterator<TimeValue> {
@@ -203,6 +231,11 @@ private static final class WrappedBackoffPolicy extends BackoffPolicy {
         public Iterator<TimeValue> iterator() {
             return new WrappedBackoffIterator(delegate.iterator(), onBackoff);
         }
+
+        @Override
+        public String toString() {
+            return "WrappedBackoffPolicy{delegate=" + delegate + ", onBackoff=" + onBackoff + '}';
+        }
     }
 
     private static final class WrappedBackoffIterator implements Iterator<TimeValue> {
@@ -228,4 +261,60 @@ public TimeValue next() {
             return delegate.next();
         }
     }
+
+    private static final class LinearBackoff extends BackoffPolicy {
+
+        private final TimeValue delayIncrement;
+        private final int maxNumberOfRetries;
+        private final TimeValue maximumDelay;
+
+        private LinearBackoff(TimeValue delayIncrement, int maxNumberOfRetries, @Nullable TimeValue maximumDelay) {
+            this.delayIncrement = delayIncrement;
+            this.maxNumberOfRetries = maxNumberOfRetries;
+            this.maximumDelay = maximumDelay;
+        }
+
+        @Override
+        public Iterator<TimeValue> iterator() {
+            return new LinearBackoffIterator(delayIncrement, maxNumberOfRetries, maximumDelay);
+        }
+
+        @Override
+        public String toString() {
+            return "LinearBackoff{"
+                + "delayIncrement="
+                + delayIncrement
+                + ", maxNumberOfRetries="
+                + maxNumberOfRetries
+                + ", maximumDelay="
+                + maximumDelay
+                + '}';
+        }
+    }
+
+    private static final class LinearBackoffIterator implements Iterator<TimeValue> {
+
+        private final TimeValue delayIncrement;
+        private final int maxNumberOfRetries;
+        private final TimeValue maximumDelay;
+        private int curr;
+
+        private LinearBackoffIterator(TimeValue delayIncrement, int maxNumberOfRetries, @Nullable TimeValue maximumDelay) {
+            this.delayIncrement = delayIncrement;
+            this.maxNumberOfRetries = maxNumberOfRetries;
+            this.maximumDelay = maximumDelay;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return curr < maxNumberOfRetries;
+        }
+
+        @Override
+        public TimeValue next() {
+            curr++;
+            TimeValue timeValue = TimeValue.timeValueMillis(curr * delayIncrement.millis());
+            return maximumDelay == null ? timeValue : timeValue.compareTo(maximumDelay) < 0 ? timeValue : maximumDelay;
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
index ae8de474daf93..926056fec3ec8 100644
--- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
+++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
@@ -66,7 +66,6 @@ public enum ReferenceDocs {
     BOOTSTRAP_CHECK_ROLE_MAPPINGS,
     BOOTSTRAP_CHECK_TLS,
     BOOTSTRAP_CHECK_TOKEN_SSL,
-    BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP,
     CONTACT_SUPPORT,
     UNASSIGNED_SHARDS,
     EXECUTABLE_JNA_TMPDIR,
@@ -82,6 +81,7 @@ public enum ReferenceDocs {
     FORMING_SINGLE_NODE_CLUSTERS,
     CIRCUIT_BREAKER_ERRORS,
     ALLOCATION_EXPLAIN_NO_COPIES,
+    ALLOCATION_EXPLAIN_MAX_RETRY,
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 
diff --git a/server/src/main/java/org/elasticsearch/common/Strings.java b/server/src/main/java/org/elasticsearch/common/Strings.java
index 4314d2e16799a..82504b5840792 100644
--- a/server/src/main/java/org/elasticsearch/common/Strings.java
+++ b/server/src/main/java/org/elasticsearch/common/Strings.java
@@ -285,6 +285,7 @@ private static String changeFirstCharacterCase(String str, boolean capitalize) {
     static final Set<Character> INVALID_CHARS = Set.of('\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',');
 
     public static final String INVALID_FILENAME_CHARS = INVALID_CHARS.stream()
+        .sorted()
         .map(c -> "'" + c + "'")
         .collect(Collectors.joining(",", "[", "]"));
 
diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
index c4240672239fa..53e3b4b4796dc 100644
--- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
+++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
@@ -19,6 +19,8 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.Iterator;
 import java.util.List;
 
@@ -56,11 +58,14 @@ public int bufferSizeInBytes() {
     public BlobContainer blobContainer(BlobPath path) {
         Path f = buildPath(path);
         if (readOnly == false) {
-            try {
-                Files.createDirectories(f);
-            } catch (IOException ex) {
-                throw new ElasticsearchException("failed to create blob container", ex);
-            }
+            AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+                try {
+                    Files.createDirectories(f);
+                } catch (IOException ex) {
+                    throw new ElasticsearchException("failed to create blob container", ex);
+                }
+                return null;
+            });
         }
         return new FsBlobContainer(this, path, f);
     }
diff --git a/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java b/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java
index 9327dbe78077f..e5c9b14cf90fc 100644
--- a/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java
+++ b/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java
@@ -109,8 +109,8 @@ public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws Circu
             if (closed) {
                 throw new IllegalStateException("already closed");
             }
-            if (preallocationUsed == preallocated) {
-                // Preallocation buffer was full before this request
+            if (preallocationUsed == preallocated || bytes == 0L) {
+                // Preallocation buffer was full before this request or we are checking the parent circuit breaker
                 next.addEstimateBytesAndMaybeBreak(bytes, label);
                 return;
             }
diff --git a/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java b/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java
index dcb28a17a9b49..41998bf974bf9 100644
--- a/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java
+++ b/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java
@@ -77,6 +77,15 @@ public AbstractFileWatchingService(Path watchedFile) {
 
     protected abstract void processInitialFileMissing() throws InterruptedException, ExecutionException, IOException;
 
+    /**
+     * Defaults to generic {@link #processFileChanges()} behavior.
+     * An implementation can override this to define different file handling when the file is processed during
+     * initial service start.
+     */
+    protected void processFileOnServiceStart() throws IOException, ExecutionException, InterruptedException {
+        processFileChanges();
+    }
+
     public final void addFileChangedListener(FileChangedListener listener) {
         eventListeners.add(listener);
     }
@@ -174,7 +183,7 @@ protected final void watcherThread() {
 
             if (Files.exists(path)) {
                 logger.debug("found initial operator settings file [{}], applying...", path);
-                processSettingsAndNotifyListeners();
+                processSettingsOnServiceStartAndNotifyListeners();
             } else {
                 processInitialFileMissing();
                 // Notify everyone we don't have any initial file settings
@@ -290,9 +299,9 @@ final WatchKey enableDirectoryWatcher(WatchKey previousKey, Path settingsDir) th
         } while (true);
     }
 
-    void processSettingsAndNotifyListeners() throws InterruptedException {
+    void processSettingsOnServiceStartAndNotifyListeners() throws InterruptedException {
         try {
-            processFileChanges();
+            processFileOnServiceStart();
             for (var listener : eventListeners) {
                 listener.watchedFileChanged();
             }
@@ -301,6 +310,25 @@ void processSettingsAndNotifyListeners() throws InterruptedException {
         }
     }
 
+    void processSettingsAndNotifyListeners() throws InterruptedException {
+        try {
+            processFileChanges();
+        } catch (IOException | ExecutionException e) {
+            onProcessFileChangesException(e);
+            return;
+        }
+        for (var listener : eventListeners) {
+            listener.watchedFileChanged();
+        }
+    }
+
+    /**
+     * Called for checked exceptions only.
+     */
+    protected void onProcessFileChangesException(Exception e) {
+        logger.error(() -> "Error processing watched file: " + watchedFile(), e);
+    }
+
     // package private for testing
     long retryDelayMillis(int failedCount) {
         assert failedCount < 31; // don't let the count overflow
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 5043508c781f0..a57b8b4d23cdb 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -88,7 +88,7 @@
 import java.util.Objects;
 
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene912";
+    public static final String LATEST_CODEC = "Lucene100";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 
@@ -392,8 +392,8 @@ public static ScoreDoc readScoreDoc(StreamInput in) throws IOException {
     private static final Class<?> GEO_DISTANCE_SORT_TYPE_CLASS = LatLonDocValuesField.newDistanceSort("some_geo_field", 0, 0).getClass();
 
     public static void writeTotalHits(StreamOutput out, TotalHits totalHits) throws IOException {
-        out.writeVLong(totalHits.value);
-        out.writeEnum(totalHits.relation);
+        out.writeVLong(totalHits.value());
+        out.writeEnum(totalHits.relation());
     }
 
     public static void writeTopDocs(StreamOutput out, TopDocsAndMaxScore topDocs) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java b/server/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
index 625438ebdff97..cbceef120b877 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOBooleanSupplier;
 import org.elasticsearch.core.Nullable;
 
 import java.io.IOException;
@@ -177,6 +178,11 @@ public boolean seekExact(BytesRef text) throws IOException {
         }
     }
 
+    @Override
+    public IOBooleanSupplier prepareSeekExact(BytesRef bytesRef) {
+        return () -> this.seekExact(bytesRef);
+    }
+
     @Override
     public int docFreq() throws IOException {
         return currentDocFreq;
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java b/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java
index 5bc52253939af..9460aba0a99cb 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java
@@ -14,7 +14,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 
 import java.util.ArrayList;
@@ -38,8 +37,6 @@ public static Automaton caseInsensitivePrefix(String s) {
         Automaton a = Operations.concatenate(list);
         // since all elements in the list should be deterministic already, the concatenation also is, so no need to determinized
         assert a.isDeterministic();
-        a = MinimizationOperations.minimize(a, 0);
-        assert a.isDeterministic();
         return a;
     }
 
@@ -100,7 +97,7 @@ public static Automaton toCaseInsensitiveWildcardAutomaton(Term wildcardquery) {
             i += length;
         }
 
-        return Operations.concatenate(automata);
+        return Operations.determinize(Operations.concatenate(automata), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     protected static Automaton toCaseInsensitiveString(BytesRef br) {
@@ -117,7 +114,6 @@ public static Automaton toCaseInsensitiveString(String s) {
         Automaton a = Operations.concatenate(list);
         // concatenating deterministic automata should result in a deterministic automaton. No need to determinize here.
         assert a.isDeterministic();
-        a = MinimizationOperations.minimize(a, 0);
         return a;
     }
 
@@ -132,7 +128,6 @@ public static Automaton toCaseInsensitiveChar(int codepoint) {
         if (altCase != codepoint) {
             result = Operations.union(case1, Automata.makeChar(altCase));
             // this automaton should always be deterministic, no need to determinize
-            result = MinimizationOperations.minimize(result, 0);
             assert result.isDeterministic();
         } else {
             result = case1;
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java
index b6f102a98203f..65688b69f5aa0 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java
@@ -20,12 +20,12 @@ public CaseInsensitivePrefixQuery(Term term) {
         super(term, caseInsensitivePrefix(term.text()));
     }
 
-    public CaseInsensitivePrefixQuery(Term term, int determinizeWorkLimit, boolean isBinary) {
-        super(term, caseInsensitivePrefix(term.text()), determinizeWorkLimit, isBinary);
+    public CaseInsensitivePrefixQuery(Term term, boolean isBinary) {
+        super(term, caseInsensitivePrefix(term.text()), isBinary);
     }
 
-    public CaseInsensitivePrefixQuery(Term term, int determinizeWorkLimit, boolean isBinary, MultiTermQuery.RewriteMethod rewriteMethod) {
-        super(term, caseInsensitivePrefix(term.text()), determinizeWorkLimit, isBinary, rewriteMethod);
+    public CaseInsensitivePrefixQuery(Term term, boolean isBinary, MultiTermQuery.RewriteMethod rewriteMethod) {
+        super(term, caseInsensitivePrefix(term.text()), isBinary, rewriteMethod);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java
index 91700e5ffe6c1..6368acf383120 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java
@@ -26,8 +26,8 @@ public CaseInsensitiveWildcardQuery(Term term) {
         super(term, toCaseInsensitiveWildcardAutomaton(term));
     }
 
-    public CaseInsensitiveWildcardQuery(Term term, int determinizeWorkLimit, boolean isBinary, RewriteMethod rewriteMethod) {
-        super(term, toCaseInsensitiveWildcardAutomaton(term), determinizeWorkLimit, isBinary, rewriteMethod);
+    public CaseInsensitiveWildcardQuery(Term term, boolean isBinary, RewriteMethod rewriteMethod) {
+        super(term, toCaseInsensitiveWildcardAutomaton(term), isBinary, rewriteMethod);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
index 25fa926ada2c8..e2ac58caccd57 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
@@ -123,7 +123,7 @@ public static Query applyMinimumShouldMatch(BooleanQuery query, @Nullable String
         }
         int optionalClauses = 0;
         for (BooleanClause c : query.clauses()) {
-            if (c.getOccur() == BooleanClause.Occur.SHOULD) {
+            if (c.occur() == BooleanClause.Occur.SHOULD) {
                 optionalClauses++;
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
index 13fae303909f5..299739fc3ba8a 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/SpanBooleanQueryRewriteWithMaxClause.java
@@ -19,7 +19,7 @@
 import org.apache.lucene.queries.spans.SpanOrQuery;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.queries.spans.SpanTermQuery;
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.AttributeSource;
@@ -42,7 +42,7 @@ public class SpanBooleanQueryRewriteWithMaxClause extends SpanMultiTermQueryWrap
     private final boolean hardLimit;
 
     public SpanBooleanQueryRewriteWithMaxClause() {
-        this(BooleanQuery.getMaxClauseCount(), true);
+        this(IndexSearcher.getMaxClauseCount(), true);
     }
 
     public SpanBooleanQueryRewriteWithMaxClause(int maxExpansions, boolean hardLimit) {
@@ -59,10 +59,11 @@ public boolean isHardLimit() {
     }
 
     @Override
-    public SpanQuery rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
+    public SpanQuery rewrite(IndexSearcher indexSearcher, MultiTermQuery query) throws IOException {
         final MultiTermQuery.RewriteMethod delegate = new MultiTermQuery.RewriteMethod() {
             @Override
-            public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
+            public Query rewrite(IndexSearcher indexSearcher, MultiTermQuery query) throws IOException {
+                IndexReader reader = indexSearcher.getIndexReader();
                 Collection<SpanQuery> queries = collectTerms(reader, query);
                 if (queries.size() == 0) {
                     return new SpanMatchNoDocsQuery(query.getField(), "no expansion found for " + query.toString());
@@ -99,7 +100,7 @@ private Collection<SpanQuery> collectTerms(IndexReader reader, MultiTermQuery qu
                                         + query.toString()
                                         + " ] "
                                         + "exceeds maxClauseCount [ Boolean maxClauseCount is set to "
-                                        + BooleanQuery.getMaxClauseCount()
+                                        + IndexSearcher.getMaxClauseCount()
                                         + "]"
                                 );
                             } else {
@@ -112,6 +113,6 @@ private Collection<SpanQuery> collectTerms(IndexReader reader, MultiTermQuery qu
                 return queries;
             }
         };
-        return (SpanQuery) delegate.rewrite(reader, query);
+        return (SpanQuery) delegate.rewrite(indexSearcher, query);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java b/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java
index f8d0c81466dcc..54cd4c9946f62 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java
@@ -34,6 +34,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.similarities.TFIDFSimilarity;
@@ -207,7 +208,7 @@ public final class XMoreLikeThis {
     /**
      * Return a Query with no more than this many terms.
      *
-     * @see BooleanQuery#getMaxClauseCount
+     * @see IndexSearcher#getMaxClauseCount
      * @see #setMaxQueryTerms
      */
     public static final int DEFAULT_MAX_QUERY_TERMS = 25;
@@ -468,7 +469,7 @@ private void addToQuery(PriorityQueue<ScoreTerm> q, BooleanQuery.Builder query)
 
             try {
                 query.add(tq, BooleanClause.Occur.SHOULD);
-            } catch (BooleanQuery.TooManyClauses ignore) {
+            } catch (IndexSearcher.TooManyClauses ignore) {
                 break;
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
index ff82160be0325..5a0c216c4e717 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -272,44 +272,65 @@ class CustomBoostFactorWeight extends Weight {
             this.needsScores = needsScores;
         }
 
-        private FunctionFactorScorer functionScorer(LeafReaderContext context) throws IOException {
-            Scorer subQueryScorer = subQueryWeight.scorer(context);
-            if (subQueryScorer == null) {
+        private ScorerSupplier functionScorerSupplier(LeafReaderContext context) throws IOException {
+            ScorerSupplier subQueryScorerSupplier = subQueryWeight.scorerSupplier(context);
+            if (subQueryScorerSupplier == null) {
                 return null;
             }
-            final long leadCost = subQueryScorer.iterator().cost();
-            final LeafScoreFunction[] leafFunctions = new LeafScoreFunction[functions.length];
-            final Bits[] docSets = new Bits[functions.length];
-            for (int i = 0; i < functions.length; i++) {
-                ScoreFunction function = functions[i];
-                leafFunctions[i] = function.getLeafScoreFunction(context);
-                if (filterWeights[i] != null) {
-                    ScorerSupplier filterScorerSupplier = filterWeights[i].scorerSupplier(context);
-                    docSets[i] = Lucene.asSequentialAccessBits(context.reader().maxDoc(), filterScorerSupplier, leadCost);
-                } else {
-                    docSets[i] = new Bits.MatchAllBits(context.reader().maxDoc());
+            return new ScorerSupplier() {
+                @Override
+                public Scorer get(long leadCost) throws IOException {
+                    Scorer subQueryScorer = subQueryScorerSupplier.get(leadCost);
+                    final LeafScoreFunction[] leafFunctions = new LeafScoreFunction[functions.length];
+                    final Bits[] docSets = new Bits[functions.length];
+                    for (int i = 0; i < functions.length; i++) {
+                        ScoreFunction function = functions[i];
+                        leafFunctions[i] = function.getLeafScoreFunction(context);
+                        if (filterWeights[i] != null) {
+                            ScorerSupplier filterScorerSupplier = filterWeights[i].scorerSupplier(context);
+                            docSets[i] = Lucene.asSequentialAccessBits(context.reader().maxDoc(), filterScorerSupplier, leadCost);
+                        } else {
+                            docSets[i] = new Bits.MatchAllBits(context.reader().maxDoc());
+                        }
+                    }
+                    return new FunctionFactorScorer(
+                        subQueryScorer,
+                        scoreMode,
+                        functions,
+                        maxBoost,
+                        leafFunctions,
+                        docSets,
+                        combineFunction,
+                        needsScores
+                    );
                 }
-            }
-            return new FunctionFactorScorer(
-                this,
-                subQueryScorer,
-                scoreMode,
-                functions,
-                maxBoost,
-                leafFunctions,
-                docSets,
-                combineFunction,
-                needsScores
-            );
+
+                @Override
+                public long cost() {
+                    return subQueryScorerSupplier.cost();
+                }
+            };
         }
 
         @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            Scorer scorer = functionScorer(context);
-            if (scorer != null && minScore != null) {
-                scorer = new MinScoreScorer(this, scorer, minScore);
+        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+            ScorerSupplier scorerSupplier = functionScorerSupplier(context);
+
+            if (scorerSupplier == null || minScore == null) {
+                return scorerSupplier;
             }
-            return scorer;
+
+            return new ScorerSupplier() {
+                @Override
+                public Scorer get(long leadCost) throws IOException {
+                    return new MinScoreScorer(scorerSupplier.get(leadCost), minScore);
+                }
+
+                @Override
+                public long cost() {
+                    return scorerSupplier.cost();
+                }
+            };
         }
 
         @Override
@@ -356,7 +377,8 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 } else if (singleFunction && functionsExplanations.size() == 1) {
                     factorExplanation = functionsExplanations.get(0);
                 } else {
-                    FunctionFactorScorer scorer = functionScorer(context);
+
+                    FunctionFactorScorer scorer = (FunctionFactorScorer) functionScorerSupplier(context).get(1L);
                     int actualDoc = scorer.iterator().advance(doc);
                     assert (actualDoc == doc);
                     double score = scorer.computeScore(doc, expl.getValue().floatValue());
@@ -391,7 +413,6 @@ static class FunctionFactorScorer extends FilterScorer {
         private final boolean needsScores;
 
         private FunctionFactorScorer(
-            CustomBoostFactorWeight w,
             Scorer scorer,
             ScoreMode scoreMode,
             ScoreFunction[] functions,
@@ -401,7 +422,7 @@ private FunctionFactorScorer(
             CombineFunction scoreCombiner,
             boolean needsScores
         ) throws IOException {
-            super(scorer, w);
+            super(scorer);
             this.scoreMode = scoreMode;
             this.functions = functions;
             this.leafFunctions = leafFunctions;
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java
index 3d23f66b09d82..0fd46447b3ea9 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/MinScoreScorer.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.search.Weight;
 
 import java.io.IOException;
 
@@ -25,12 +24,11 @@ public final class MinScoreScorer extends Scorer {
     private float curScore;
     private final float boost;
 
-    public MinScoreScorer(Weight weight, Scorer scorer, float minScore) {
-        this(weight, scorer, minScore, 1f);
+    public MinScoreScorer(Scorer scorer, float minScore) {
+        this(scorer, minScore, 1f);
     }
 
-    public MinScoreScorer(Weight weight, Scorer scorer, float minScore, float boost) {
-        super(weight);
+    public MinScoreScorer(Scorer scorer, float minScore, float boost) {
         this.in = scorer;
         this.minScore = minScore;
         this.boost = boost;
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
index 4222b5dff98ab..d38243f5348c4 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
@@ -27,14 +27,8 @@
 public class ScriptScoreFunction extends ScoreFunction {
 
     static final class CannedScorer extends Scorable {
-        protected int docid;
         protected float score;
 
-        @Override
-        public int docID() {
-            return docid;
-        }
-
         @Override
         public float score() {
             return score;
@@ -70,14 +64,13 @@ public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) throws IOEx
 
         if (script.needs_termStats()) {
             assert termStatsFactory != null;
-            leafScript._setTermStats(termStatsFactory.apply(ctx, scorer::docID));
+            leafScript._setTermStats(termStatsFactory.apply(ctx, leafScript::docId));
         }
 
         return new LeafScoreFunction() {
 
             private double score(int docId, float subQueryScore, ScoreScript.ExplanationHolder holder) throws IOException {
                 leafScript.setDocument(docId);
-                scorer.docid = docId;
                 scorer.score = subQueryScore;
                 double result = leafScript.execute(holder);
 
@@ -97,7 +90,6 @@ public Explanation explainScore(int docId, Explanation subQueryScore) throws IOE
                 Explanation exp;
                 if (leafScript instanceof ExplainableScoreScript) {
                     leafScript.setDocument(docId);
-                    scorer.docid = docId;
                     scorer.score = subQueryScore.getValue().floatValue();
                     exp = ((ExplainableScoreScript) leafScript).explain(subQueryScore);
                 } else {
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java
index 5e3f8e8e62714..e58b2fffed001 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
@@ -38,6 +39,7 @@
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.IntSupplier;
 
 /**
  * A query that uses a script to compute documents' scores.
@@ -104,30 +106,40 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
         }
 
         return new Weight(this) {
-            @Override
-            public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
-                if (minScore == null) {
-                    final BulkScorer subQueryBulkScorer = subQueryWeight.bulkScorer(context);
-                    if (subQueryBulkScorer == null) {
-                        return null;
-                    }
-                    return new ScriptScoreBulkScorer(subQueryBulkScorer, subQueryScoreMode, makeScoreScript(context), boost);
-                } else {
-                    return super.bulkScorer(context);
-                }
-            }
 
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                Scorer subQueryScorer = subQueryWeight.scorer(context);
-                if (subQueryScorer == null) {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                ScorerSupplier subQueryScorerSupplier = subQueryWeight.scorerSupplier(context);
+                if (subQueryScorerSupplier == null) {
                     return null;
                 }
-                Scorer scriptScorer = new ScriptScorer(this, makeScoreScript(context), subQueryScorer, subQueryScoreMode, boost, null);
-                if (minScore != null) {
-                    scriptScorer = new MinScoreScorer(this, scriptScorer, minScore);
-                }
-                return scriptScorer;
+
+                return new ScorerSupplier() {
+                    @Override
+                    public Scorer get(long leadCost) throws IOException {
+                        Scorer subQueryScorer = subQueryScorerSupplier.get(leadCost);
+                        Scorer scriptScorer = new ScriptScorer(makeScoreScript(context), subQueryScorer, subQueryScoreMode, boost, null);
+                        if (minScore != null) {
+                            scriptScorer = new MinScoreScorer(scriptScorer, minScore);
+                        }
+                        return scriptScorer;
+                    }
+
+                    @Override
+                    public BulkScorer bulkScorer() throws IOException {
+                        if (minScore == null) {
+                            final BulkScorer subQueryBulkScorer = subQueryScorerSupplier.bulkScorer();
+                            return new ScriptScoreBulkScorer(subQueryBulkScorer, subQueryScoreMode, makeScoreScript(context), boost);
+                        } else {
+                            return super.bulkScorer();
+                        }
+                    }
+
+                    @Override
+                    public long cost() {
+                        return subQueryScorerSupplier.cost();
+                    }
+                };
             }
 
             @Override
@@ -138,7 +150,6 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 }
                 ExplanationHolder explanationHolder = new ExplanationHolder();
                 Scorer scorer = new ScriptScorer(
-                    this,
                     makeScoreScript(context),
                     subQueryWeight.scorer(context),
                     subQueryScoreMode,
@@ -231,14 +242,12 @@ private static class ScriptScorer extends Scorer {
         private final ExplanationHolder explanation;
 
         ScriptScorer(
-            Weight weight,
             ScoreScript scoreScript,
             Scorer subQueryScorer,
             ScoreMode subQueryScoreMode,
             float boost,
             ExplanationHolder explanation
         ) {
-            super(weight);
             this.scoreScript = scoreScript;
             if (subQueryScoreMode == ScoreMode.COMPLETE) {
                 scoreScript.setScorer(subQueryScorer);
@@ -292,19 +301,27 @@ private static class ScriptScorable extends Scorable {
         private final ScoreScript scoreScript;
         private final Scorable subQueryScorer;
         private final float boost;
+        private final IntSupplier docIDSupplier;
 
-        ScriptScorable(ScoreScript scoreScript, Scorable subQueryScorer, ScoreMode subQueryScoreMode, float boost) {
+        ScriptScorable(
+            ScoreScript scoreScript,
+            Scorable subQueryScorer,
+            ScoreMode subQueryScoreMode,
+            float boost,
+            IntSupplier docIDSupplier
+        ) {
             this.scoreScript = scoreScript;
             if (subQueryScoreMode == ScoreMode.COMPLETE) {
                 scoreScript.setScorer(subQueryScorer);
             }
             this.subQueryScorer = subQueryScorer;
             this.boost = boost;
+            this.docIDSupplier = docIDSupplier;
         }
 
         @Override
         public float score() throws IOException {
-            int docId = docID();
+            int docId = docIDSupplier.getAsInt();
             scoreScript.setDocument(docId);
             float score = (float) scoreScript.execute(null);
             if (score < 0f || Float.isNaN(score)) {
@@ -320,10 +337,6 @@ public float score() throws IOException {
             return score * boost;
         }
 
-        @Override
-        public int docID() {
-            return subQueryScorer.docID();
-        }
     }
 
     /**
@@ -350,9 +363,18 @@ public int score(LeafCollector collector, Bits acceptDocs, int min, int max) thr
 
         private LeafCollector wrapCollector(LeafCollector collector) {
             return new FilterLeafCollector(collector) {
+
+                private int docID;
+
                 @Override
                 public void setScorer(Scorable scorer) throws IOException {
-                    in.setScorer(new ScriptScorable(scoreScript, scorer, subQueryScoreMode, boost));
+                    in.setScorer(new ScriptScorable(scoreScript, scorer, subQueryScoreMode, boost, () -> docID));
+                }
+
+                @Override
+                public void collect(int doc) throws IOException {
+                    this.docID = doc;
+                    super.collect(doc);
                 }
             };
         }
diff --git a/server/src/main/java/org/elasticsearch/common/regex/Regex.java b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
index d5b2e8497fc0b..aaaab78b71736 100644
--- a/server/src/main/java/org/elasticsearch/common/regex/Regex.java
+++ b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
@@ -69,7 +69,7 @@ public static Automaton simpleMatchToAutomaton(String pattern) {
             previous = i + 1;
         }
         automata.add(Automata.makeString(pattern.substring(previous)));
-        return Operations.concatenate(automata);
+        return Operations.determinize(Operations.concatenate(automata), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     /**
@@ -113,7 +113,7 @@ public static Automaton simpleMatchToAutomaton(String... patterns) {
             prefixAutomaton.add(Automata.makeAnyString());
             automata.add(Operations.concatenate(prefixAutomaton));
         }
-        return Operations.union(automata);
+        return Operations.determinize(Operations.union(automata), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 8cbacccb915ac..7bb78eabc8727 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -108,6 +108,7 @@
 import org.elasticsearch.monitor.process.ProcessService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeRoleSettings;
+import org.elasticsearch.node.ShutdownPrepareService;
 import org.elasticsearch.persistent.PersistentTasksClusterService;
 import org.elasticsearch.persistent.decider.EnableAssignmentDecider;
 import org.elasticsearch.plugins.PluginsService;
@@ -456,6 +457,8 @@ public void apply(Settings value, Settings current, Settings previous) {
         Environment.PATH_SHARED_DATA_SETTING,
         NodeEnvironment.NODE_ID_SEED_SETTING,
         Node.INITIAL_STATE_TIMEOUT_SETTING,
+        ShutdownPrepareService.MAXIMUM_SHUTDOWN_TIMEOUT_SETTING,
+        ShutdownPrepareService.MAXIMUM_REINDEXING_TIMEOUT_SETTING,
         DiscoveryModule.DISCOVERY_TYPE_SETTING,
         DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING,
         DiscoveryModule.ELECTION_STRATEGY_SETTING,
diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index ad3d7d7f1c2ec..f5276bbe49b63 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryFactory;
 import org.elasticsearch.index.store.Store;
@@ -186,6 +187,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING,
         IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING,
         IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
+        IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING,
+        SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING,
 
         // validate that built-in similarities don't get redefined
         Setting.groupSetting("index.similarity.", (s) -> {
diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index 232ce34b153ab..defaddb25eb47 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -254,7 +254,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException {
         }
 
         Directory directory = new NIOFSDirectory(configDir);
-        try (ChecksumIndexInput input = directory.openChecksumInput(KEYSTORE_FILENAME, IOContext.READONCE)) {
+        try (ChecksumIndexInput input = directory.openChecksumInput(KEYSTORE_FILENAME)) {
             final int formatVersion;
             try {
                 formatVersion = CodecUtil.checkHeader(input, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, CURRENT_VERSION);
diff --git a/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java b/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java
index e4f1608a52d15..4a2e1cd92d4da 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java
@@ -11,11 +11,11 @@
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.reservedstate.service.ReservedStateVersion;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -130,7 +130,7 @@ public LocallyMountedSecrets(Environment environment) {
                 throw new IllegalStateException("Error processing secrets file", e);
             }
         } else {
-            secrets.set(new LocalFileSecrets(Map.of(), new ReservedStateVersion(-1L, Version.CURRENT)));
+            secrets.set(new LocalFileSecrets(Map.of(), new ReservedStateVersion(-1L, BuildVersion.current())));
         }
         this.secretsDir = secretsDirPath.toString();
         this.secretsFile = secretsFilePath.toString();
diff --git a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
index 208d29edad71d..288462ba3bbcb 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
@@ -48,7 +48,7 @@ public BytesRefHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         boolean success = false;
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            this.hashes = bigArrays.newIntArray(capacity, false);
+            this.hashes = bigArrays.newIntArray(maxSize, false);
             this.bytesRefs = new BytesRefArray(capacity, bigArrays);
             success = true;
         } finally {
@@ -98,7 +98,7 @@ public BytesRefHash(BytesRefArray bytesRefs, float maxLoadFactor, BigArrays bigA
         boolean success = false;
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            this.hashes = bigArrays.newIntArray(bytesRefs.size() + 1, false);
+            this.hashes = bigArrays.newIntArray(maxSize, false);
             this.bytesRefs = BytesRefArray.takeOwnershipOf(bytesRefs);
             success = true;
         } finally {
@@ -182,7 +182,6 @@ private long set(BytesRef key, int code, long id) {
     private void append(long id, BytesRef key, int code) {
         assert size == id;
         bytesRefs.append(key);
-        hashes = bigArrays.grow(hashes, id + 1);
         hashes.set(id, code);
     }
 
@@ -211,6 +210,7 @@ public long add(BytesRef key, int code) {
         if (size >= maxSize) {
             assert size == maxSize;
             grow();
+            hashes = bigArrays.resize(hashes, maxSize);
         }
         assert size < maxSize;
         return set(key, rehash(code), size);
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongHash.java
index 0c681063c50b0..3eeb60e419a19 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongHash.java
@@ -33,7 +33,7 @@ public LongHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            keys = bigArrays.newLongArray(capacity, false);
+            keys = bigArrays.newLongArray(maxSize, false);
         } finally {
             if (keys == null) {
                 close();
@@ -78,7 +78,6 @@ private long set(long key, long id) {
     }
 
     private void append(long id, long key) {
-        keys = bigArrays.grow(keys, id + 1);
         keys.set(id, key);
     }
 
@@ -102,6 +101,7 @@ public long add(long key) {
         if (size >= maxSize) {
             assert size == maxSize;
             grow();
+            keys = bigArrays.resize(keys, maxSize);
         }
         assert size < maxSize;
         return set(key, size);
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
index f7708af59dde2..031794ed9c9c6 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
@@ -40,7 +40,7 @@ public LongLongHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            keys = bigArrays.newLongArray(2 * capacity, false);
+            keys = bigArrays.newLongArray(2 * maxSize, false);
         } finally {
             if (keys == null) {
                 close();
@@ -99,7 +99,6 @@ private long set(long key1, long key2, long id) {
 
     private void append(long id, long key1, long key2) {
         long keyOffset = 2 * id;
-        keys = bigArrays.grow(keys, keyOffset + 2);
         keys.set(keyOffset, key1);
         keys.set(keyOffset + 1, key2);
     }
@@ -128,6 +127,7 @@ public long add(long key1, long key2) {
         if (size >= maxSize) {
             assert size == maxSize;
             grow();
+            keys = bigArrays.resize(keys, maxSize * 2);
         }
         assert size < maxSize;
         return set(key1, key2, size);
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 7c001a39a306d..a9e13b86a5159 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -101,7 +101,7 @@ public ThreadContext(Settings settings) {
     }
 
     /**
-     * Removes the current context and resets a default context. The removed context can be
+     * Removes the current context and resets a default context except for headers involved in task tracing. The removed context can be
      * restored by closing the returned {@link StoredContext}.
      * @return a stored context that will restore the current context to its state at the point this method was called
      */
@@ -159,6 +159,28 @@ public StoredContext stashContextPreservingRequestHeaders(final String... reques
         return stashContextPreservingRequestHeaders(Set.of(requestHeaders));
     }
 
+    /**
+     * Removes the current context and replaces it with a completely empty default context, detaching execution entirely from the calling
+     * context. The calling context can be restored by closing the returned {@link StoredContext}. Similar to {@link #stashContext()} except
+     * that this method does not even preserve tracing-related headers.
+     */
+    public StoredContext newEmptyContext() {
+        final var callingContext = threadLocal.get();
+        threadLocal.set(DEFAULT_CONTEXT);
+        return storedOriginalContext(callingContext);
+    }
+
+    /**
+     * Removes the current context and replaces it with a completely empty system context, detaching execution entirely from the calling
+     * context. The calling context can be restored by closing the returned {@link StoredContext}. Similar to {@link #stashContext()} except
+     * that this method does not even preserve tracing-related headers.
+     */
+    public StoredContext newEmptySystemContext() {
+        final var callingContext = threadLocal.get();
+        threadLocal.set(DEFAULT_CONTEXT.setSystemContext());
+        return storedOriginalContext(callingContext);
+    }
+
     /**
      * When using a {@link org.elasticsearch.telemetry.tracing.Tracer} to capture activity in Elasticsearch, when a parent span is already
      * in progress, it is necessary to start a new context before beginning a child span. This method creates a context,
@@ -330,6 +352,16 @@ public StoredContext newStoredContextPreservingResponseHeaders() {
         };
     }
 
+    /**
+     * Capture the current context and then restore the given context, returning a {@link StoredContext} that reverts back to the current
+     * context again. Equivalent to using {@link #newStoredContext()} and then calling {@code existingContext.restore()}.
+     */
+    public StoredContext restoreExistingContext(StoredContext existingContext) {
+        final var originalContext = threadLocal.get();
+        existingContext.restore();
+        return storedOriginalContext(originalContext);
+    }
+
     /**
      * Just like {@link #stashContext()} but no default context is set.
      */
@@ -914,14 +946,13 @@ private class ContextPreservingRunnable implements WrappedRunnable {
         private final ThreadContext.StoredContext ctx;
 
         private ContextPreservingRunnable(Runnable in) {
-            ctx = newStoredContext();
+            this.ctx = newStoredContext();
             this.in = in;
         }
 
         @Override
         public void run() {
-            try (ThreadContext.StoredContext ignore = stashContext()) {
-                ctx.restore();
+            try (var ignore = restoreExistingContext(ctx)) {
                 in.run();
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
index 63859d89f3e22..db0b5b4357c7d 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
@@ -34,6 +34,27 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) {
         return new ChunkedToXContentBuilder(params);
     }
 
+    /**
+     * Create an iterator of {@link ToXContent} chunks for a REST response for the given {@link RestApiVersion}. Each chunk is serialized
+     * with the same {@link XContentBuilder} and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed
+     * as the {@code params} argument. For best results, all chunks should be {@code O(1)} size. The last chunk in the iterator must always
+     * yield at least one byte of output. See also {@link ChunkedToXContentHelper} for some handy utilities.
+     * <p>
+     * Note that chunked response bodies cannot send deprecation warning headers once transmission has started, so implementations must
+     * check for deprecated feature use before returning.
+     * <p>
+     * By default, delegates to {@link #toXContentChunked} or {#toXContentChunkedV8}.
+     *
+     * @return iterator over chunks of {@link ToXContent}
+     */
+    default Iterator<? extends ToXContent> toXContentChunked(RestApiVersion restApiVersion, ToXContent.Params params) {
+        return switch (restApiVersion) {
+            case V_7 -> throw new AssertionError("v7 API not supported");
+            case V_8 -> toXContentChunkedV8(params);
+            case V_9 -> toXContentChunked(params);
+        };
+    }
+
     /**
      * Create an iterator of {@link ToXContent} chunks for a REST response. Each chunk is serialized with the same {@link XContentBuilder}
      * and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed as the {@code params} argument. For
@@ -48,12 +69,12 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) {
     Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params);
 
     /**
-     * Create an iterator of {@link ToXContent} chunks for a response to the {@link RestApiVersion#V_7} API. Each chunk is serialized with
+     * Create an iterator of {@link ToXContent} chunks for a response to the {@link RestApiVersion#V_8} API. Each chunk is serialized with
      * the same {@link XContentBuilder} and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed as the
      * {@code params} argument. For best results, all chunks should be {@code O(1)} size. The last chunk in the iterator must always yield
      * at least one byte of output. See also {@link ChunkedToXContentHelper} for some handy utilities.
      * <p>
-     * Similar to {@link #toXContentChunked} but for the {@link RestApiVersion#V_7} API. By default this method delegates to {@link
+     * Similar to {@link #toXContentChunked} but for the {@link RestApiVersion#V_8} API. By default this method delegates to {@link
      * #toXContentChunked}.
      * <p>
      * Note that chunked response bodies cannot send deprecation warning headers once transmission has started, so implementations must
@@ -61,7 +82,7 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) {
      *
      * @return iterator over chunks of {@link ToXContent}
      */
-    default Iterator<? extends ToXContent> toXContentChunkedV7(ToXContent.Params params) {
+    default Iterator<? extends ToXContent> toXContentChunkedV8(ToXContent.Params params) {
         return toXContentChunked(params);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java
index 0102e58c7c1dc..a3141bff7c6e2 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java
@@ -248,7 +248,7 @@ private void endArray() {
         addChunk((b, p) -> b.endArray());
     }
 
-    public ChunkedToXContentBuilder array(String name, String... values) {
+    public ChunkedToXContentBuilder array(String name, String[] values) {
         addChunk((b, p) -> b.array(name, values));
         return this;
     }
@@ -350,6 +350,26 @@ public ChunkedToXContentBuilder field(String name, Long value) {
         return this;
     }
 
+    public ChunkedToXContentBuilder field(String name, float value) {
+        addChunk((b, p) -> b.field(name, value));
+        return this;
+    }
+
+    public ChunkedToXContentBuilder field(String name, Float value) {
+        addChunk((b, p) -> b.field(name, value));
+        return this;
+    }
+
+    public ChunkedToXContentBuilder field(String name, double value) {
+        addChunk((b, p) -> b.field(name, value));
+        return this;
+    }
+
+    public ChunkedToXContentBuilder field(String name, Double value) {
+        addChunk((b, p) -> b.field(name, value));
+        return this;
+    }
+
     public ChunkedToXContentBuilder field(String name, String value) {
         addChunk((b, p) -> b.field(name, value));
         return this;
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
index fcbe0ac2b2edb..2e78cc6f516b1 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
@@ -53,10 +53,6 @@ public static Iterator<ToXContent> field(String name, String value) {
         return Iterators.single(((builder, params) -> builder.field(name, value)));
     }
 
-    public static Iterator<ToXContent> array(String name, Iterator<? extends ToXContent> contents) {
-        return Iterators.concat(ChunkedToXContentHelper.startArray(name), contents, ChunkedToXContentHelper.endArray());
-    }
-
     /**
      * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link
      * Iterators#single}, but still useful because it avoids any type ambiguity.
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java
index dea851b1b553a..0298e1a123b58 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java
@@ -57,13 +57,13 @@ public Map<Class<?>, XContentBuilder.Writer> getXContentWriters() {
         // Fully-qualified here to reduce ambiguity around our (ES') Version class
         writers.put(org.apache.lucene.util.Version.class, (b, v) -> b.value(Objects.toString(v)));
         writers.put(TimeValue.class, (b, v) -> b.value(v.toString()));
-        writers.put(ZonedDateTime.class, XContentBuilder::timeValue);
-        writers.put(OffsetDateTime.class, XContentBuilder::timeValue);
-        writers.put(OffsetTime.class, XContentBuilder::timeValue);
-        writers.put(java.time.Instant.class, XContentBuilder::timeValue);
-        writers.put(LocalDateTime.class, XContentBuilder::timeValue);
-        writers.put(LocalDate.class, XContentBuilder::timeValue);
-        writers.put(LocalTime.class, XContentBuilder::timeValue);
+        writers.put(ZonedDateTime.class, XContentBuilder::timestampValue);
+        writers.put(OffsetDateTime.class, XContentBuilder::timestampValue);
+        writers.put(OffsetTime.class, XContentBuilder::timestampValue);
+        writers.put(java.time.Instant.class, XContentBuilder::timestampValue);
+        writers.put(LocalDateTime.class, XContentBuilder::timestampValue);
+        writers.put(LocalDate.class, XContentBuilder::timestampValue);
+        writers.put(LocalTime.class, XContentBuilder::timestampValue);
         writers.put(DayOfWeek.class, (b, v) -> b.value(v.toString()));
         writers.put(Month.class, (b, v) -> b.value(v.toString()));
         writers.put(MonthDay.class, (b, v) -> b.value(v.toString()));
@@ -103,10 +103,8 @@ public Map<Class<?>, XContentBuilder.HumanReadableTransformer> getXContentHumanR
     public Map<Class<?>, Function<Object, Object>> getDateTransformers() {
         Map<Class<?>, Function<Object, Object>> transformers = new HashMap<>();
         transformers.put(Date.class, d -> DEFAULT_FORMATTER.format(((Date) d).toInstant()));
-        transformers.put(Long.class, d -> DEFAULT_FORMATTER.format(Instant.ofEpochMilli((long) d)));
         transformers.put(Calendar.class, d -> DEFAULT_FORMATTER.format(((Calendar) d).toInstant()));
         transformers.put(GregorianCalendar.class, d -> DEFAULT_FORMATTER.format(((Calendar) d).toInstant()));
-        transformers.put(Instant.class, d -> DEFAULT_FORMATTER.format((Instant) d));
         transformers.put(ZonedDateTime.class, d -> DEFAULT_FORMATTER.format((ZonedDateTime) d));
         transformers.put(OffsetDateTime.class, d -> DEFAULT_FORMATTER.format((OffsetDateTime) d));
         transformers.put(OffsetTime.class, d -> OFFSET_TIME_FORMATTER.format((OffsetTime) d));
@@ -119,4 +117,9 @@ public Map<Class<?>, Function<Object, Object>> getDateTransformers() {
         transformers.put(LocalTime.class, d -> LOCAL_TIME_FORMATTER.format((LocalTime) d));
         return transformers;
     }
+
+    @Override
+    public String formatUnixEpochMillis(long unixEpochMillis) {
+        return DEFAULT_FORMATTER.format(Instant.ofEpochMilli(unixEpochMillis));
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
index 2ef96123e63d8..c4b03c712c272 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
@@ -280,8 +280,8 @@ public static Function<Map<String, Object>, Map<String, Object>> filter(String[]
             include = matchAllAutomaton;
         } else {
             Automaton includeA = Regex.simpleMatchToAutomaton(includes);
-            includeA = makeMatchDotsInFieldNames(includeA);
-            include = new CharacterRunAutomaton(includeA, MAX_DETERMINIZED_STATES);
+            includeA = Operations.determinize(makeMatchDotsInFieldNames(includeA), MAX_DETERMINIZED_STATES);
+            include = new CharacterRunAutomaton(includeA);
         }
 
         Automaton excludeA;
@@ -289,9 +289,9 @@ public static Function<Map<String, Object>, Map<String, Object>> filter(String[]
             excludeA = Automata.makeEmpty();
         } else {
             excludeA = Regex.simpleMatchToAutomaton(excludes);
-            excludeA = makeMatchDotsInFieldNames(excludeA);
+            excludeA = Operations.determinize(makeMatchDotsInFieldNames(excludeA), MAX_DETERMINIZED_STATES);
         }
-        CharacterRunAutomaton exclude = new CharacterRunAutomaton(excludeA, MAX_DETERMINIZED_STATES);
+        CharacterRunAutomaton exclude = new CharacterRunAutomaton(excludeA);
 
         // NOTE: We cannot use Operations.minus because of the special case that
         // we want all sub properties to match as soon as an object matches
diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java
index 0de346249ccbc..5536b06d4d587 100644
--- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java
+++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java
@@ -58,14 +58,6 @@ public abstract class BuildVersion {
      */
     public abstract boolean isFutureVersion();
 
-    // temporary
-    // TODO[wrb]: remove from PersistedClusterStateService
-    // TODO[wrb]: remove from security bootstrap checks
-    @Deprecated
-    public Version toVersion() {
-        return null;
-    }
-
     /**
      * Create a {@link BuildVersion} from a version ID number.
      *
@@ -80,6 +72,16 @@ public static BuildVersion fromVersionId(int versionId) {
         return CurrentExtensionHolder.BUILD_EXTENSION.fromVersionId(versionId);
     }
 
+    /**
+     * Create a {@link BuildVersion} from a version string.
+     *
+     * @param version A string representation of a version
+     * @return a version representing a build or release of Elasticsearch
+     */
+    public static BuildVersion fromString(String version) {
+        return CurrentExtensionHolder.BUILD_EXTENSION.fromString(version);
+    }
+
     /**
      * Get the current build version.
      *
@@ -118,6 +120,11 @@ public BuildVersion currentBuildVersion() {
         public BuildVersion fromVersionId(int versionId) {
             return new DefaultBuildVersion(versionId);
         }
+
+        @Override
+        public BuildVersion fromString(String version) {
+            return new DefaultBuildVersion(version);
+        }
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java b/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java
index e0531b5a192a0..9cf0d60719653 100644
--- a/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java
+++ b/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java
@@ -28,15 +28,17 @@ final class DefaultBuildVersion extends BuildVersion {
 
     public static BuildVersion CURRENT = new DefaultBuildVersion(Version.CURRENT.id());
 
-    private final int versionId;
     private final Version version;
 
     DefaultBuildVersion(int versionId) {
         assert versionId >= 0 : "Release version IDs must be non-negative integers";
-        this.versionId = versionId;
         this.version = Version.fromId(versionId);
     }
 
+    DefaultBuildVersion(String version) {
+        this.version = Version.fromString(Objects.requireNonNull(version));
+    }
+
     @Override
     public boolean onOrAfterMinimumCompatible() {
         return Version.CURRENT.minimumCompatibilityVersion().onOrBefore(version);
@@ -49,12 +51,7 @@ public boolean isFutureVersion() {
 
     @Override
     public int id() {
-        return versionId;
-    }
-
-    @Override
-    public Version toVersion() {
-        return version;
+        return version.id();
     }
 
     @Override
@@ -62,16 +59,16 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         DefaultBuildVersion that = (DefaultBuildVersion) o;
-        return versionId == that.versionId;
+        return version.equals(that.version);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(versionId);
+        return Objects.hash(version.id());
     }
 
     @Override
     public String toString() {
-        return Version.fromId(versionId).toString();
+        return version.toString();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/env/NodeMetadata.java b/server/src/main/java/org/elasticsearch/env/NodeMetadata.java
index 6a72a7e7fcda5..5b2ee39c1b622 100644
--- a/server/src/main/java/org/elasticsearch/env/NodeMetadata.java
+++ b/server/src/main/java/org/elasticsearch/env/NodeMetadata.java
@@ -42,7 +42,6 @@ public final class NodeMetadata {
 
     private final IndexVersion oldestIndexVersion;
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // version should be non-null in the node metadata from v9 onwards
     private NodeMetadata(
         final String nodeId,
         final BuildVersion buildVersion,
@@ -112,11 +111,7 @@ public IndexVersion oldestIndexVersion() {
         return oldestIndexVersion;
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
     public void verifyUpgradeToCurrentVersion() {
-        // Enable the following assertion for V9:
-        // assert (nodeVersion.equals(BuildVersion.empty()) == false) : "version is required in the node metadata from v9 onwards";
-
         if (nodeVersion.onOrAfterMinimumCompatible() == false) {
             throw new IllegalStateException(
                 "cannot upgrade a node from version ["
diff --git a/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java b/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java
index 96158965cddfe..1ddc8d5b26bd9 100644
--- a/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java
+++ b/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java
@@ -74,7 +74,7 @@ protected void processDataPaths(Terminal terminal, Path[] paths, OptionSet optio
                 "found ["
                     + nodeMetadata
                     + "] which is compatible with current version ["
-                    + Version.CURRENT
+                    + BuildVersion.current()
                     + "], so there is no need to override the version checks"
             );
         } catch (IllegalStateException e) {
@@ -86,10 +86,10 @@ protected void processDataPaths(Terminal terminal, Path[] paths, OptionSet optio
             (nodeMetadata.nodeVersion().onOrAfterMinimumCompatible() == false ? TOO_OLD_MESSAGE : TOO_NEW_MESSAGE).replace(
                 "V_OLD",
                 nodeMetadata.nodeVersion().toString()
-            ).replace("V_NEW", nodeMetadata.nodeVersion().toString()).replace("V_CUR", Version.CURRENT.toString())
+            ).replace("V_NEW", nodeMetadata.nodeVersion().toString()).replace("V_CUR", BuildVersion.current().toString())
         );
 
-        PersistedClusterStateService.overrideVersion(Version.CURRENT, paths);
+        PersistedClusterStateService.overrideVersion(BuildVersion.current(), paths);
 
         terminal.println(SUCCESS_MESSAGE);
     }
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
index c863a5bac973a..a7baca59e1857 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
-import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -33,8 +32,6 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
 import org.elasticsearch.core.IOUtils;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.env.NodeMetadata;
 import org.elasticsearch.index.IndexVersions;
@@ -185,16 +182,6 @@ private PersistedState createOnDiskPersistedState(
         long lastAcceptedVersion = onDiskState.lastAcceptedVersion;
         long currentTerm = onDiskState.currentTerm;
 
-        if (onDiskState.empty()) {
-            @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // legacy metadata loader is not needed anymore from v9 onwards
-            final Tuple<Manifest, Metadata> legacyState = metaStateService.loadFullState();
-            if (legacyState.v1().isEmpty() == false) {
-                metadata = legacyState.v2();
-                lastAcceptedVersion = legacyState.v1().clusterStateVersion();
-                currentTerm = legacyState.v1().currentTerm();
-            }
-        }
-
         PersistedState persistedState = null;
         boolean success = false;
         try {
diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java
index 4260ef51a3976..5f07deff31eea 100644
--- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java
@@ -12,22 +12,17 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Predicate;
 
 /**
@@ -45,118 +40,6 @@ public MetaStateService(NodeEnvironment nodeEnv, NamedXContentRegistry namedXCon
         this.namedXContentRegistry = namedXContentRegistry;
     }
 
-    /**
-     * Loads the full state, which includes both the global state and all the indices meta data. <br>
-     * When loading, manifest file is consulted (represented by {@link Manifest} class), to load proper generations. <br>
-     * If there is no manifest file on disk, this method fallbacks to BWC mode, where latest generation of global and indices
-     * metadata is loaded. Please note that currently there is no way to distinguish between manifest file being removed and manifest
-     * file was not yet created. It means that this method always fallbacks to BWC mode, if there is no manifest file.
-     *
-     * @return tuple of {@link Manifest} and {@link Metadata} with global metadata and indices metadata. If there is no state on disk,
-     * meta state with globalGeneration -1 and empty meta data is returned.
-     * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file.
-     */
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    public Tuple<Manifest, Metadata> loadFullState() throws IOException {
-        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths());
-        if (manifest == null) {
-            return loadFullStateBWC();
-        }
-
-        final Metadata.Builder metadataBuilder;
-        if (manifest.isGlobalGenerationMissing()) {
-            metadataBuilder = Metadata.builder();
-        } else {
-            final Metadata globalMetadata = Metadata.FORMAT.loadGeneration(
-                logger,
-                namedXContentRegistry,
-                manifest.globalGeneration(),
-                nodeEnv.nodeDataPaths()
-            );
-            if (globalMetadata != null) {
-                metadataBuilder = Metadata.builder(globalMetadata);
-            } else {
-                throw new IOException("failed to find global metadata [generation: " + manifest.globalGeneration() + "]");
-            }
-        }
-
-        for (Map.Entry<Index, Long> entry : manifest.indexGenerations().entrySet()) {
-            final Index index = entry.getKey();
-            final long generation = entry.getValue();
-            final String indexFolderName = index.getUUID();
-            final IndexMetadata indexMetadata = IndexMetadata.FORMAT.loadGeneration(
-                logger,
-                namedXContentRegistry,
-                generation,
-                nodeEnv.resolveIndexFolder(indexFolderName)
-            );
-            if (indexMetadata != null) {
-                metadataBuilder.put(indexMetadata, false);
-            } else {
-                throw new IOException(
-                    "failed to find metadata for existing index "
-                        + index.getName()
-                        + " [location: "
-                        + indexFolderName
-                        + ", generation: "
-                        + generation
-                        + "]"
-                );
-            }
-        }
-
-        return new Tuple<>(manifest, metadataBuilder.build());
-    }
-
-    /**
-     * "Manifest-less" BWC version of loading metadata from disk. See also {@link #loadFullState()}
-     */
-    private Tuple<Manifest, Metadata> loadFullStateBWC() throws IOException {
-        Map<Index, Long> indices = new HashMap<>();
-        Metadata.Builder metadataBuilder;
-
-        Tuple<Metadata, Long> metadataAndGeneration = Metadata.FORMAT.loadLatestStateWithGeneration(
-            logger,
-            namedXContentRegistry,
-            nodeEnv.nodeDataPaths()
-        );
-        Metadata globalMetadata = metadataAndGeneration.v1();
-        long globalStateGeneration = metadataAndGeneration.v2();
-
-        final IndexGraveyard indexGraveyard;
-        if (globalMetadata != null) {
-            metadataBuilder = Metadata.builder(globalMetadata);
-            indexGraveyard = globalMetadata.custom(IndexGraveyard.TYPE);
-        } else {
-            metadataBuilder = Metadata.builder();
-            indexGraveyard = IndexGraveyard.builder().build();
-        }
-
-        for (String indexFolderName : nodeEnv.availableIndexFolders()) {
-            Tuple<IndexMetadata, Long> indexMetadataAndGeneration = IndexMetadata.FORMAT.loadLatestStateWithGeneration(
-                logger,
-                namedXContentRegistry,
-                nodeEnv.resolveIndexFolder(indexFolderName)
-            );
-            IndexMetadata indexMetadata = indexMetadataAndGeneration.v1();
-            long generation = indexMetadataAndGeneration.v2();
-            if (indexMetadata != null) {
-                if (indexGraveyard.containsIndex(indexMetadata.getIndex())) {
-                    logger.debug("[{}] found metadata for deleted index [{}]", indexFolderName, indexMetadata.getIndex());
-                    // this index folder is cleared up when state is recovered
-                } else {
-                    indices.put(indexMetadata.getIndex(), generation);
-                    metadataBuilder.put(indexMetadata, false);
-                }
-            } else {
-                logger.debug("[{}] failed to find metadata for existing index location", indexFolderName);
-            }
-        }
-
-        Manifest manifest = Manifest.unknownCurrentTermAndVersion(globalStateGeneration, indices);
-        return new Tuple<>(manifest, metadataBuilder.build());
-    }
-
     /**
      * Loads the index state for the provided index name, returning null if doesn't exists.
      */
@@ -193,7 +76,7 @@ List<IndexMetadata> loadIndicesStates(Predicate<String> excludeIndexPathIdsPredi
     }
 
     /**
-     * Loads the global state, *without* index state, see {@link #loadFullState()} for that.
+     * Loads the global state, *without* index state
      */
     Metadata loadGlobalState() throws IOException {
         return Metadata.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths());
diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java
index 30b8d72b83f4c..3e68ec5243f5f 100644
--- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java
+++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.transport.Transports;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -485,6 +486,7 @@ public Tuple<T, Long> loadLatestStateWithGeneration(Logger logger, NamedXContent
      * @param dataLocations the data-locations to try.
      * @return the latest state or <code>null</code> if no state was found.
      */
+    @Nullable
     public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException {
         return loadLatestStateWithGeneration(logger, namedXContentRegistry, dataLocations).v1();
     }
diff --git a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
index 749946e05b745..92b8686700a05 100644
--- a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -41,7 +42,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
@@ -158,8 +158,6 @@ public class PersistedClusterStateService {
     public static final int IS_LAST_PAGE = 1;
     public static final int IS_NOT_LAST_PAGE = 0;
     private static final int COMMIT_DATA_SIZE = 7;
-    // We added CLUSTER_UUID_KEY and CLUSTER_UUID_COMMITTED_KEY in 8.8
-    private static final int COMMIT_DATA_SIZE_BEFORE_8_8 = 5;
 
     private static final MergePolicy NO_MERGE_POLICY = noMergePolicy();
     private static final MergePolicy DEFAULT_MERGE_POLICY = defaultMergePolicy();
@@ -349,7 +347,7 @@ public record OnDiskStateMetadata(
     @Nullable
     public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException {
         String nodeId = null;
-        Version version = null;
+        BuildVersion version = null;
         IndexVersion oldestIndexVersion = IndexVersions.ZERO;
         for (final Path dataPath : dataPaths) {
             final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME);
@@ -366,7 +364,7 @@ public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException {
                         );
                     } else if (nodeId == null) {
                         nodeId = thisNodeId;
-                        version = Version.fromId(Integer.parseInt(userData.get(NODE_VERSION_KEY)));
+                        version = BuildVersion.fromVersionId(Integer.parseInt(userData.get(NODE_VERSION_KEY)));
                         if (userData.containsKey(OLDEST_INDEX_VERSION_KEY)) {
                             oldestIndexVersion = IndexVersion.fromId(Integer.parseInt(userData.get(OLDEST_INDEX_VERSION_KEY)));
                         } else {
@@ -381,14 +379,13 @@ public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException {
         if (nodeId == null) {
             return null;
         }
-        // TODO: remove use of Version here (ES-7343)
-        return new NodeMetadata(nodeId, BuildVersion.fromVersionId(version.id()), oldestIndexVersion);
+        return new NodeMetadata(nodeId, version, oldestIndexVersion);
     }
 
     /**
      * Overrides the version field for the metadata in the given data path
      */
-    public static void overrideVersion(Version newVersion, Path... dataPaths) throws IOException {
+    public static void overrideVersion(BuildVersion newVersion, Path... dataPaths) throws IOException {
         for (final Path dataPath : dataPaths) {
             final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME);
             if (Files.exists(indexPath)) {
@@ -398,7 +395,7 @@ public static void overrideVersion(Version newVersion, Path... dataPaths) throws
 
                     try (IndexWriter indexWriter = createIndexWriter(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)), true)) {
                         final Map<String, String> commitData = new HashMap<>(userData);
-                        commitData.put(NODE_VERSION_KEY, Integer.toString(newVersion.id));
+                        commitData.put(NODE_VERSION_KEY, Integer.toString(newVersion.id()));
                         commitData.put(OVERRIDDEN_NODE_VERSION_KEY, Boolean.toString(true));
                         indexWriter.setLiveCommitData(commitData.entrySet());
                         indexWriter.commit();
@@ -449,7 +446,7 @@ OnDiskState loadBestOnDiskState(boolean checkClean) throws IOException {
                                 // resources during test execution
                                 checkIndex.setThreadCount(1);
                                 checkIndex.setInfoStream(printStream);
-                                checkIndex.setChecksumsOnly(true);
+                                checkIndex.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_CHECKSUM_CHECKS);
                                 isClean = checkIndex.checkIndex().clean;
                             }
 
@@ -663,11 +660,9 @@ public OnDiskStateMetadata loadOnDiskStateMetadataFromUserData(Map<String, Strin
         assert userData.get(LAST_ACCEPTED_VERSION_KEY) != null;
         assert userData.get(NODE_ID_KEY) != null;
         assert userData.get(NODE_VERSION_KEY) != null;
-        var nodeVersion = Version.fromId(Integer.parseInt(userData.get(NODE_VERSION_KEY)));
-        assert userData.get(CLUSTER_UUID_KEY) != null || nodeVersion.before(Version.V_8_8_0);
-        assert userData.get(CLUSTER_UUID_COMMITTED_KEY) != null || nodeVersion.before(Version.V_8_8_0);
-        assert userData.get(OVERRIDDEN_NODE_VERSION_KEY) != null
-            || userData.size() == (nodeVersion.onOrAfter(Version.V_8_8_0) ? COMMIT_DATA_SIZE : COMMIT_DATA_SIZE_BEFORE_8_8) : userData;
+        assert userData.get(CLUSTER_UUID_KEY) != null;
+        assert userData.get(CLUSTER_UUID_COMMITTED_KEY) != null;
+        assert userData.get(OVERRIDDEN_NODE_VERSION_KEY) != null || userData.size() == COMMIT_DATA_SIZE;
         return new OnDiskStateMetadata(
             Long.parseLong(userData.get(CURRENT_TERM_KEY)),
             Long.parseLong(userData.get(LAST_ACCEPTED_VERSION_KEY)),
@@ -705,10 +700,11 @@ private static void consumeFromType(
                 final Bits liveDocs = leafReaderContext.reader().getLiveDocs();
                 final IntPredicate isLiveDoc = liveDocs == null ? i -> true : liveDocs::get;
                 final DocIdSetIterator docIdSetIterator = scorer.iterator();
+                final StoredFields storedFields = leafReaderContext.reader().storedFields();
                 while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
                     if (isLiveDoc.test(docIdSetIterator.docID())) {
                         logger.trace("processing doc {}", docIdSetIterator.docID());
-                        final Document document = leafReaderContext.reader().document(docIdSetIterator.docID());
+                        final Document document = storedFields.document(docIdSetIterator.docID());
                         final BytesArray documentData = new BytesArray(document.getBinaryValue(DATA_FIELD_NAME));
 
                         if (document.getField(PAGE_FIELD_NAME) == null) {
@@ -856,7 +852,7 @@ void prepareCommit(
             final Map<String, String> commitData = Maps.newMapWithExpectedSize(COMMIT_DATA_SIZE);
             commitData.put(CURRENT_TERM_KEY, Long.toString(currentTerm));
             commitData.put(LAST_ACCEPTED_VERSION_KEY, Long.toString(lastAcceptedVersion));
-            commitData.put(NODE_VERSION_KEY, Integer.toString(Version.CURRENT.id));
+            commitData.put(NODE_VERSION_KEY, Integer.toString(BuildVersion.current().id()));
             commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id()));
             commitData.put(NODE_ID_KEY, nodeId);
             commitData.put(CLUSTER_UUID_KEY, clusterUUID);
diff --git a/server/src/main/java/org/elasticsearch/health/Diagnosis.java b/server/src/main/java/org/elasticsearch/health/Diagnosis.java
index 41301e2d52a53..b1af4a1c383da 100644
--- a/server/src/main/java/org/elasticsearch/health/Diagnosis.java
+++ b/server/src/main/java/org/elasticsearch/health/Diagnosis.java
@@ -10,14 +10,12 @@
 package org.elasticsearch.health;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
@@ -78,22 +76,20 @@ public Resource(Collection<DiscoveryNode> nodes) {
         }
 
         @Override
-        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
-            final Iterator<? extends ToXContent> valuesIterator;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+            var builder = ChunkedToXContent.builder(params);
             if (nodes != null) {
-                valuesIterator = Iterators.map(nodes.iterator(), node -> (builder, params) -> {
-                    builder.startObject();
-                    builder.field(ID_FIELD, node.getId());
+                return builder.array(type.displayValue, nodes.iterator(), node -> (b, p) -> {
+                    b.startObject();
+                    b.field(ID_FIELD, node.getId());
                     if (node.getName() != null) {
-                        builder.field(NAME_FIELD, node.getName());
+                        b.field(NAME_FIELD, node.getName());
                     }
-                    builder.endObject();
-                    return builder;
+                    return b.endObject();
                 });
             } else {
-                valuesIterator = Iterators.map(values.iterator(), value -> (builder, params) -> builder.value(value));
+                return builder.array(type.displayValue, values.toArray(String[]::new));
             }
-            return ChunkedToXContentHelper.array(type.displayValue, valuesIterator);
         }
 
         @Override
@@ -144,30 +140,18 @@ public String getUniqueId() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
-        final Iterator<? extends ToXContent> resourcesIterator;
-        if (affectedResources == null) {
-            resourcesIterator = Collections.emptyIterator();
-        } else {
-            resourcesIterator = Iterators.flatMap(affectedResources.iterator(), s -> s.toXContentChunked(outerParams));
-        }
-        return Iterators.concat(Iterators.single((ToXContent) (builder, params) -> {
-            builder.startObject();
-            builder.field("id", definition.getUniqueId());
-            builder.field("cause", definition.cause);
-            builder.field("action", definition.action);
-            builder.field("help_url", definition.helpURL);
-
-            if (affectedResources != null && affectedResources.size() > 0) {
-                builder.startObject("affected_resources");
-            }
-            return builder;
-        }), resourcesIterator, Iterators.single((builder, params) -> {
-            if (affectedResources != null && affectedResources.size() > 0) {
-                builder.endObject();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContent.builder(params).object(ob -> {
+            ob.append((b, p) -> {
+                b.field("id", definition.getUniqueId());
+                b.field("cause", definition.cause);
+                b.field("action", definition.action);
+                b.field("help_url", definition.helpURL);
+                return b;
+            });
+            if (affectedResources != null && affectedResources.isEmpty() == false) {
+                ob.object("affected_resources", affectedResources.iterator(), ChunkedToXContentBuilder::append);
             }
-            builder.endObject();
-            return builder;
-        }));
+        });
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java b/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
index 6944ac74c8115..1a84abd9f7c16 100644
--- a/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
+++ b/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
@@ -9,11 +9,11 @@
 
 package org.elasticsearch.health;
 
-import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.xcontent.ToXContent;
 
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -26,33 +26,22 @@ public record HealthIndicatorResult(
     List<Diagnosis> diagnosisList
 ) implements ChunkedToXContentObject {
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
-        final Iterator<? extends ToXContent> diagnosisIterator;
-        if (diagnosisList == null) {
-            diagnosisIterator = Collections.emptyIterator();
-        } else {
-            diagnosisIterator = Iterators.flatMap(diagnosisList.iterator(), s -> s.toXContentChunked(outerParams));
-        }
-        return Iterators.concat(Iterators.single((ToXContent) (builder, params) -> {
-            builder.startObject();
-            builder.field("status", status.xContentValue());
-            builder.field("symptom", symptom);
-            if (details != null && HealthIndicatorDetails.EMPTY.equals(details) == false) {
-                builder.field("details", details, params);
-            }
-            if (impacts != null && impacts.isEmpty() == false) {
-                builder.field("impacts", impacts);
-            }
-            if (diagnosisList != null && diagnosisList.isEmpty() == false) {
-                builder.startArray("diagnosis");
-            }
-            return builder;
-        }), diagnosisIterator, Iterators.single((builder, params) -> {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContent.builder(params).object(ob -> {
+            ob.append((b, p) -> {
+                b.field("status", status.xContentValue());
+                b.field("symptom", symptom);
+                if (details != null && HealthIndicatorDetails.EMPTY.equals(details) == false) {
+                    b.field("details", details, p);
+                }
+                if (impacts != null && impacts.isEmpty() == false) {
+                    b.field("impacts", impacts);
+                }
+                return b;
+            });
             if (diagnosisList != null && diagnosisList.isEmpty() == false) {
-                builder.endArray();
+                ob.array("diagnosis", diagnosisList.iterator(), ChunkedToXContentBuilder::append);
             }
-            builder.endObject();
-            return builder;
-        }));
+        });
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java
index 5dfd698b2bb20..e6339344b6e5f 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexMode.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java
@@ -9,7 +9,9 @@
 
 package org.elasticsearch.index;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService;
 import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -37,8 +39,10 @@
 import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.BooleanSupplier;
@@ -75,7 +79,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
         }
 
         @Override
-        public CompressedXContent getDefaultMapping() {
+        public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
             return null;
         }
 
@@ -120,8 +124,8 @@ public boolean shouldValidateTimestamp() {
         public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {}
 
         @Override
-        public boolean isSyntheticSourceEnabled() {
-            return false;
+        public SourceFieldMapper.Mode defaultSourceMode() {
+            return SourceFieldMapper.Mode.STORED;
         }
     },
     TIME_SERIES("time_series") {
@@ -171,7 +175,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
         }
 
         @Override
-        public CompressedXContent getDefaultMapping() {
+        public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
             return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
         }
 
@@ -217,14 +221,14 @@ public boolean shouldValidateTimestamp() {
 
         @Override
         public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
-            if (sourceFieldMapper.isSynthetic() == false) {
-                throw new IllegalArgumentException("time series indices only support synthetic source");
+            if (sourceFieldMapper.enabled() == false) {
+                throw new IllegalArgumentException("_source can not be disabled in index using [" + IndexMode.TIME_SERIES + "] index mode");
             }
         }
 
         @Override
-        public boolean isSyntheticSourceEnabled() {
-            return true;
+        public SourceFieldMapper.Mode defaultSourceMode() {
+            return SourceFieldMapper.Mode.SYNTHETIC;
         }
     },
     LOGSDB("logsdb") {
@@ -249,8 +253,10 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
         }
 
         @Override
-        public CompressedXContent getDefaultMapping() {
-            return DEFAULT_LOGS_TIMESTAMP_MAPPING;
+        public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
+            return indexSettings != null && indexSettings.getIndexSortConfig().hasPrimarySortOnField(HOST_NAME)
+                ? DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME
+                : DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
         }
 
         @Override
@@ -292,22 +298,96 @@ public boolean shouldValidateTimestamp() {
 
         @Override
         public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
-            if (sourceFieldMapper.isSynthetic() == false) {
-                throw new IllegalArgumentException("Indices with with index mode [" + IndexMode.LOGSDB + "] only support synthetic source");
+            if (sourceFieldMapper.enabled() == false) {
+                throw new IllegalArgumentException("_source can not be disabled in index using [" + IndexMode.LOGSDB + "] index mode");
             }
         }
 
         @Override
-        public boolean isSyntheticSourceEnabled() {
-            return true;
+        public SourceFieldMapper.Mode defaultSourceMode() {
+            return SourceFieldMapper.Mode.SYNTHETIC;
         }
 
         @Override
         public String getDefaultCodec() {
             return CodecService.BEST_COMPRESSION_CODEC;
         }
+    },
+    LOOKUP("lookup") {
+        @Override
+        void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
+            final Integer providedNumberOfShards = (Integer) settings.get(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING);
+            if (providedNumberOfShards != null && providedNumberOfShards != 1) {
+                throw new IllegalArgumentException(
+                    "index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided " + providedNumberOfShards
+                );
+            }
+        }
+
+        @Override
+        public void validateMapping(MappingLookup lookup) {};
+
+        @Override
+        public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) {}
+
+        @Override
+        public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) {
+
+        }
+
+        @Override
+        public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
+            return null;
+        }
+
+        @Override
+        public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) {
+            return null;
+        }
+
+        @Override
+        public MetadataFieldMapper timeSeriesIdFieldMapper() {
+            // non time-series indices must not have a TimeSeriesIdFieldMapper
+            return null;
+        }
+
+        @Override
+        public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() {
+            // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper
+            return null;
+        }
+
+        @Override
+        public IdFieldMapper idFieldMapperWithoutFieldData() {
+            return ProvidedIdFieldMapper.NO_FIELD_DATA;
+        }
+
+        @Override
+        public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) {
+            return new ProvidedIdFieldMapper(fieldDataEnabled);
+        }
+
+        @Override
+        public DocumentDimensions buildDocumentDimensions(IndexSettings settings) {
+            return DocumentDimensions.Noop.INSTANCE;
+        }
+
+        @Override
+        public boolean shouldValidateTimestamp() {
+            return false;
+        }
+
+        @Override
+        public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {}
+
+        @Override
+        public SourceFieldMapper.Mode defaultSourceMode() {
+            return SourceFieldMapper.Mode.STORED;
+        }
     };
 
+    private static final String HOST_NAME = "host.name";
+
     private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
         settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
         settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
@@ -324,48 +404,33 @@ protected static String tsdbMode() {
         return "[" + IndexSettings.MODE.getKey() + "=time_series]";
     }
 
-    public static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
+    private static CompressedXContent createDefaultMapping(boolean includeHostName) throws IOException {
+        return new CompressedXContent((builder, params) -> {
+            builder.startObject(MapperService.SINGLE_MAPPING_NAME)
+                .startObject(DataStreamTimestampFieldMapper.NAME)
+                .field("enabled", true)
+                .endObject()
+                .startObject("properties")
+                .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
+                .field("type", DateFieldMapper.CONTENT_TYPE)
+                .endObject();
+
+            if (includeHostName) {
+                builder.startObject(HOST_NAME).field("type", KeywordFieldMapper.CONTENT_TYPE).field("ignore_above", 1024).endObject();
+            }
 
-    static {
-        try {
-            DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = new CompressedXContent(
-                ((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
-                    .startObject(DataStreamTimestampFieldMapper.NAME)
-                    .field("enabled", true)
-                    .endObject()
-                    .startObject("properties")
-                    .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
-                    .field("type", DateFieldMapper.CONTENT_TYPE)
-                    .field("ignore_malformed", "false")
-                    .endObject()
-                    .endObject()
-                    .endObject())
-            );
-        } catch (IOException e) {
-            throw new AssertionError(e);
-        }
+            return builder.endObject().endObject();
+        });
     }
 
-    public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING;
+    private static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
+
+    private static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME;
 
     static {
         try {
-            DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent(
-                ((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
-                    .startObject(DataStreamTimestampFieldMapper.NAME)
-                    .field("enabled", true)
-                    .endObject()
-                    .startObject("properties")
-                    .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
-                    .field("type", DateFieldMapper.CONTENT_TYPE)
-                    .endObject()
-                    .startObject("host.name")
-                    .field("type", KeywordFieldMapper.CONTENT_TYPE)
-                    .field("ignore_above", 1024)
-                    .endObject()
-                    .endObject()
-                    .endObject())
-            );
+            DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = createDefaultMapping(false);
+            DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME = createDefaultMapping(true);
         } catch (IOException e) {
             throw new AssertionError(e);
         }
@@ -381,6 +446,7 @@ protected static String tsdbMode() {
     static final List<Setting<?>> VALIDATE_WITH_SETTINGS = List.copyOf(
         Stream.concat(
             Stream.of(
+                IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING,
                 IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING,
                 IndexMetadata.INDEX_ROUTING_PATH,
                 IndexSettings.TIME_SERIES_START_TIME,
@@ -421,7 +487,7 @@ public String getName() {
      * Get default mapping for this index or {@code null} if there is none.
      */
     @Nullable
-    public abstract CompressedXContent getDefaultMapping();
+    public abstract CompressedXContent getDefaultMapping(IndexSettings indexSettings);
 
     /**
      * Build the {@link FieldMapper} for {@code _id}.
@@ -471,9 +537,9 @@ public String getName() {
     public abstract void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper);
 
     /**
-     * @return whether synthetic source is the only allowed source mode.
+     * @return default source mode for this mode
      */
-    public abstract boolean isSyntheticSourceEnabled();
+    public abstract SourceFieldMapper.Mode defaultSourceMode();
 
     public String getDefaultCodec() {
         return CodecService.DEFAULT_CODEC;
@@ -487,11 +553,12 @@ public static IndexMode fromString(String value) {
             case "standard" -> IndexMode.STANDARD;
             case "time_series" -> IndexMode.TIME_SERIES;
             case "logsdb" -> IndexMode.LOGSDB;
+            case "lookup" -> IndexMode.LOOKUP;
             default -> throw new IllegalArgumentException(
                 "["
                     + value
                     + "] is an invalid index mode, valid modes are: ["
-                    + Arrays.stream(IndexMode.values()).map(IndexMode::toString).collect(Collectors.joining())
+                    + Arrays.stream(IndexMode.values()).map(IndexMode::toString).collect(Collectors.joining(","))
                     + "]"
             );
         };
@@ -503,6 +570,7 @@ public static IndexMode readFrom(StreamInput in) throws IOException {
             case 0 -> STANDARD;
             case 1 -> TIME_SERIES;
             case 2 -> LOGSDB;
+            case 3 -> LOOKUP;
             default -> throw new IllegalStateException("unexpected index mode [" + mode + "]");
         };
     }
@@ -512,6 +580,7 @@ public static void writeTo(IndexMode indexMode, StreamOutput out) throws IOExcep
             case STANDARD -> 0;
             case TIME_SERIES -> 1;
             case LOGSDB -> 2;
+            case LOOKUP -> out.getTransportVersion().onOrAfter(TransportVersions.INDEX_MODE_LOOKUP) ? 3 : 0;
         };
         out.writeByte((byte) code);
     }
@@ -520,4 +589,37 @@ public static void writeTo(IndexMode indexMode, StreamOutput out) throws IOExcep
     public String toString() {
         return getName();
     }
+
+    /**
+     * A built-in index setting provider that supplies additional index settings based on the index mode.
+     * Currently, only the lookup index mode provides non-empty additional settings.
+     */
+    public static final class IndexModeSettingsProvider implements IndexSettingProvider {
+        @Override
+        public Settings getAdditionalIndexSettings(
+            String indexName,
+            String dataStreamName,
+            IndexMode templateIndexMode,
+            Metadata metadata,
+            Instant resolvedAt,
+            Settings indexTemplateAndCreateRequestSettings,
+            List<CompressedXContent> combinedTemplateMappings
+        ) {
+            IndexMode indexMode = templateIndexMode;
+            if (indexMode == null) {
+                String modeName = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
+                if (modeName != null) {
+                    indexMode = IndexMode.valueOf(modeName.toUpperCase(Locale.ROOT));
+                }
+            }
+            if (indexMode == LOOKUP) {
+                return Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all")
+                    .build();
+            } else {
+                return Settings.EMPTY;
+            }
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java
index 7eed5f2b7759d..4ff7ef60cc0a2 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -18,7 +18,6 @@
 import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.internal.Client;
@@ -451,7 +450,7 @@ public boolean match(String setting) {
     }
 
     public static Type defaultStoreType(final boolean allowMmap) {
-        if (allowMmap && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
+        if (allowMmap && Constants.JRE_IS_64BIT) {
             return Type.HYBRIDFS;
         } else {
             return Type.NIOFS;
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java
index aaa4c738c0e13..6a553d5dc5440 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java
@@ -30,20 +30,21 @@ public interface IndexSettingProvider {
      * Returns explicitly set default index {@link Settings} for the given index. This should not
      * return null.
      *
-     * @param indexName                The name of the new index being created
-     * @param dataStreamName           The name of the data stream if the index being created is part of a data stream otherwise
-     *                                 <code>null</code>
-     * @param isTimeSeries             Whether the template is in time series mode.
-     * @param metadata                 The current metadata instance that doesn't yet contain the index to be created
-     * @param resolvedAt               The time the request to create this new index was accepted.
-     * @param indexTemplateAndCreateRequestSettings    All the settings resolved from the template that matches and any settings
-     *                                                 defined on the create index request
-     * @param combinedTemplateMappings All the mappings resolved from the template that matches
+     * @param indexName                             The name of the new index being created
+     * @param dataStreamName                        The name of the data stream if the index being created is part of a data stream
+     *                                              otherwise <code>null</code>
+     * @param templateIndexMode                     The index mode defined in template if template creates data streams,
+     *                                              otherwise <code>null</code> is returned.
+     * @param metadata                              The current metadata instance that doesn't yet contain the index to be created
+     * @param resolvedAt                            The time the request to create this new index was accepted.
+     * @param indexTemplateAndCreateRequestSettings All the settings resolved from the template that matches and any settings
+     *                                              defined on the create index request
+     * @param combinedTemplateMappings              All the mappings resolved from the template that matches
      */
     Settings getAdditionalIndexSettings(
         String indexName,
         @Nullable String dataStreamName,
-        boolean isTimeSeries,
+        @Nullable IndexMode templateIndexMode,
         Metadata metadata,
         Instant resolvedAt,
         Settings indexTemplateAndCreateRequestSettings,
@@ -56,4 +57,15 @@ Settings getAdditionalIndexSettings(
     record Parameters(CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory) {
 
     }
+
+    /**
+     * Indicates whether the additional settings that this provider returns can overrule the settings defined in matching template
+     * or in create index request.
+     *
+     * Note that this is not used during index template validation, to avoid overruling template settings that may apply to
+     * different contexts (e.g. the provider is not used, or it returns different setting values).
+     */
+    default boolean overrulesTemplateAndRequestSettings() {
+        return false;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index e82f9eff7d5e0..25e9c1e3701fb 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -28,7 +28,9 @@
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.node.Node;
 
@@ -651,6 +653,13 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
+    public static final Setting<Boolean> SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting(
+        "index.synthetic_source.enable_second_doc_parsing_pass",
+        true,
+        Property.IndexScope,
+        Property.Dynamic
+    );
+
     /**
      * Returns <code>true</code> if TSDB encoding is enabled. The default is <code>true</code>
      */
@@ -820,6 +829,9 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private volatile long mappingDimensionFieldsLimit;
     private volatile boolean skipIgnoredSourceWrite;
     private volatile boolean skipIgnoredSourceRead;
+    private volatile boolean syntheticSourceSecondDocParsingPassEnabled;
+    private final SourceFieldMapper.Mode indexMappingSourceMode;
+    private final boolean recoverySourceEnabled;
 
     /**
      * The maximum number of refresh listeners allows on this shard.
@@ -980,6 +992,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING);
         skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
         skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
+        syntheticSourceSecondDocParsingPassEnabled = scopedSettings.get(SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING);
+        indexMappingSourceMode = scopedSettings.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
+        recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings);
 
         scopedSettings.addSettingsUpdateConsumer(
             MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,
@@ -1067,6 +1082,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
             this::setSkipIgnoredSourceWrite
         );
         scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead);
+        scopedSettings.addSettingsUpdateConsumer(
+            SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING,
+            this::setSyntheticSourceSecondDocParsingPassEnabled
+        );
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1659,6 +1678,26 @@ private void setSkipIgnoredSourceRead(boolean value) {
         this.skipIgnoredSourceRead = value;
     }
 
+    private void setSyntheticSourceSecondDocParsingPassEnabled(boolean syntheticSourceSecondDocParsingPassEnabled) {
+        this.syntheticSourceSecondDocParsingPassEnabled = syntheticSourceSecondDocParsingPassEnabled;
+    }
+
+    public boolean isSyntheticSourceSecondDocParsingPassEnabled() {
+        return syntheticSourceSecondDocParsingPassEnabled;
+    }
+
+    public SourceFieldMapper.Mode getIndexMappingSourceMode() {
+        return indexMappingSourceMode;
+    }
+
+    /**
+     * @return Whether recovery source should be enabled if needed.
+     *         Note that this is a node setting, and this setting is not sourced from index settings.
+     */
+    public boolean isRecoverySourceEnabled() {
+        return recoverySourceEnabled;
+    }
+
     /**
      * The bounds for {@code @timestamp} on this index or
      * {@code null} if there are no bounds.
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
index 7e04a64e74cb5..efb1facc79b3a 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.core.UpdateForV9;
 
 import java.lang.reflect.Field;
+import java.text.ParseException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -48,29 +49,38 @@ private static IndexVersion def(int id, Version luceneVersion) {
         return new IndexVersion(id, luceneVersion);
     }
 
+    // TODO: this is just a hack to allow to keep the V7 IndexVersion constants, during compilation. Remove
+    private static Version parseUnchecked(String version) {
+        try {
+            return Version.parse(version);
+        } catch (ParseException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove the index versions with which v9 will not need to interact
     public static final IndexVersion ZERO = def(0, Version.LATEST);
-    public static final IndexVersion V_7_0_0 = def(7_00_00_99, Version.LUCENE_8_0_0);
-
-    public static final IndexVersion V_7_1_0 = def(7_01_00_99, Version.LUCENE_8_0_0);
-    public static final IndexVersion V_7_2_0 = def(7_02_00_99, Version.LUCENE_8_0_0);
-    public static final IndexVersion V_7_2_1 = def(7_02_01_99, Version.LUCENE_8_0_0);
-    public static final IndexVersion V_7_3_0 = def(7_03_00_99, Version.LUCENE_8_1_0);
-    public static final IndexVersion V_7_4_0 = def(7_04_00_99, Version.LUCENE_8_2_0);
-    public static final IndexVersion V_7_5_0 = def(7_05_00_99, Version.LUCENE_8_3_0);
-    public static final IndexVersion V_7_5_2 = def(7_05_02_99, Version.LUCENE_8_3_0);
-    public static final IndexVersion V_7_6_0 = def(7_06_00_99, Version.LUCENE_8_4_0);
-    public static final IndexVersion V_7_7_0 = def(7_07_00_99, Version.LUCENE_8_5_1);
-    public static final IndexVersion V_7_8_0 = def(7_08_00_99, Version.LUCENE_8_5_1);
-    public static final IndexVersion V_7_9_0 = def(7_09_00_99, Version.LUCENE_8_6_0);
-    public static final IndexVersion V_7_10_0 = def(7_10_00_99, Version.LUCENE_8_7_0);
-    public static final IndexVersion V_7_11_0 = def(7_11_00_99, Version.LUCENE_8_7_0);
-    public static final IndexVersion V_7_12_0 = def(7_12_00_99, Version.LUCENE_8_8_0);
-    public static final IndexVersion V_7_13_0 = def(7_13_00_99, Version.LUCENE_8_8_2);
-    public static final IndexVersion V_7_14_0 = def(7_14_00_99, Version.LUCENE_8_9_0);
-    public static final IndexVersion V_7_15_0 = def(7_15_00_99, Version.LUCENE_8_9_0);
-    public static final IndexVersion V_7_16_0 = def(7_16_00_99, Version.LUCENE_8_10_1);
-    public static final IndexVersion V_7_17_0 = def(7_17_00_99, Version.LUCENE_8_11_1);
+
+    public static final IndexVersion V_7_0_0 = def(7_00_00_99, parseUnchecked("8.0.0"));
+    public static final IndexVersion V_7_1_0 = def(7_01_00_99, parseUnchecked("8.0.0"));
+    public static final IndexVersion V_7_2_0 = def(7_02_00_99, parseUnchecked("8.0.0"));
+    public static final IndexVersion V_7_2_1 = def(7_02_01_99, parseUnchecked("8.0.0"));
+    public static final IndexVersion V_7_3_0 = def(7_03_00_99, parseUnchecked("8.1.0"));
+    public static final IndexVersion V_7_4_0 = def(7_04_00_99, parseUnchecked("8.2.0"));
+    public static final IndexVersion V_7_5_0 = def(7_05_00_99, parseUnchecked("8.3.0"));
+    public static final IndexVersion V_7_5_2 = def(7_05_02_99, parseUnchecked("8.3.0"));
+    public static final IndexVersion V_7_6_0 = def(7_06_00_99, parseUnchecked("8.4.0"));
+    public static final IndexVersion V_7_7_0 = def(7_07_00_99, parseUnchecked("8.5.1"));
+    public static final IndexVersion V_7_8_0 = def(7_08_00_99, parseUnchecked("8.5.1"));
+    public static final IndexVersion V_7_9_0 = def(7_09_00_99, parseUnchecked("8.6.0"));
+    public static final IndexVersion V_7_10_0 = def(7_10_00_99, parseUnchecked("8.7.0"));
+    public static final IndexVersion V_7_11_0 = def(7_11_00_99, parseUnchecked("8.7.0"));
+    public static final IndexVersion V_7_12_0 = def(7_12_00_99, parseUnchecked("8.8.0"));
+    public static final IndexVersion V_7_13_0 = def(7_13_00_99, parseUnchecked("8.8.2"));
+    public static final IndexVersion V_7_14_0 = def(7_14_00_99, parseUnchecked("8.9.0"));
+    public static final IndexVersion V_7_15_0 = def(7_15_00_99, parseUnchecked("8.9.0"));
+    public static final IndexVersion V_7_16_0 = def(7_16_00_99, parseUnchecked("8.10.1"));
+    public static final IndexVersion V_7_17_0 = def(7_17_00_99, parseUnchecked("8.11.1"));
     public static final IndexVersion V_8_0_0 = def(8_00_00_99, Version.LUCENE_9_0_0);
     public static final IndexVersion V_8_1_0 = def(8_01_00_99, Version.LUCENE_9_0_0);
     public static final IndexVersion V_8_2_0 = def(8_02_00_99, Version.LUCENE_9_1_0);
@@ -118,6 +128,9 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0);
+
+    public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
+
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
diff --git a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
index 5792cafb91b77..5277999271984 100644
--- a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
+++ b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -58,6 +58,8 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 
+import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
+
 /**
  * This is a cache for {@link BitDocIdSet} based filters and is unbounded by size or time.
  * <p>
@@ -103,7 +105,10 @@ static boolean shouldLoadRandomAccessFiltersEagerly(IndexSettings settings) {
         boolean loadFiltersEagerlySetting = settings.getValue(INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING);
         boolean isStateless = DiscoveryNode.isStateless(settings.getNodeSettings());
         if (isStateless) {
-            return loadFiltersEagerlySetting && DiscoveryNode.hasRole(settings.getNodeSettings(), DiscoveryNodeRole.SEARCH_ROLE);
+            return loadFiltersEagerlySetting
+                && (DiscoveryNode.hasRole(settings.getNodeSettings(), DiscoveryNodeRole.SEARCH_ROLE)
+                    || (DiscoveryNode.hasRole(settings.getNodeSettings(), DiscoveryNodeRole.INDEX_ROLE)
+                        && INDEX_FAST_REFRESH_SETTING.get(settings.getSettings())));
         } else {
             return loadFiltersEagerlySetting;
         }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
index 144b99abe5644..c1c392ac07f18 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.Nullable;
@@ -46,7 +46,7 @@ public class CodecService implements CodecProvider {
     public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) {
         final var codecs = new HashMap<String, Codec>();
 
-        Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, bigArrays);
+        Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_SPEED, mapperService, bigArrays);
         if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
             codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays));
         } else {
@@ -58,7 +58,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays)
             BEST_COMPRESSION_CODEC,
             new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
         );
-        Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
+        Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
         codecs.put(LEGACY_BEST_COMPRESSION_CODEC, legacyBestCompressionCodec);
 
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
diff --git a/server/src/main/java/org/elasticsearch/index/codec/DeduplicatingFieldInfosFormat.java b/server/src/main/java/org/elasticsearch/index/codec/DeduplicatingFieldInfosFormat.java
index 2ba169583b712..00614140e237a 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/DeduplicatingFieldInfosFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/DeduplicatingFieldInfosFormat.java
@@ -49,11 +49,12 @@ public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segm
             deduplicated[i++] = new FieldInfo(
                 FieldMapper.internFieldName(fi.getName()),
                 fi.number,
-                fi.hasVectors(),
+                fi.hasTermVectors(),
                 fi.omitsNorms(),
                 fi.hasPayloads(),
                 fi.getIndexOptions(),
                 fi.getDocValuesType(),
+                fi.docValuesSkipIndexType(),
                 fi.getDocValuesGen(),
                 internStringStringMap(fi.attributes()),
                 fi.getPointDimensionCount(),
diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java
index 27ff19a9d8e40..9f46050f68f99 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java
@@ -9,12 +9,12 @@
 
 package org.elasticsearch.index.codec;
 
+import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch900Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch900Codec.java
new file mode 100644
index 0000000000000..4154a242c15ed
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch900Codec.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec;
+
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
+
+/**
+ * Elasticsearch codec as of 9.0. This extends the Lucene 10.0 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See
+ * {@link Zstd814StoredFieldsFormat}.
+ */
+public class Elasticsearch900Codec extends CodecService.DeduplicateFieldInfosCodec {
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    private final PostingsFormat defaultPostingsFormat;
+    private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+        @Override
+        public PostingsFormat getPostingsFormatForField(String field) {
+            return Elasticsearch900Codec.this.getPostingsFormatForField(field);
+        }
+    };
+
+    private final DocValuesFormat defaultDVFormat;
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return Elasticsearch900Codec.this.getDocValuesFormatForField(field);
+        }
+    };
+
+    private final KnnVectorsFormat defaultKnnVectorsFormat;
+    private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() {
+        @Override
+        public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+            return Elasticsearch900Codec.this.getKnnVectorsFormatForField(field);
+        }
+    };
+
+    /** Public no-arg constructor, needed for SPI loading at read-time. */
+    public Elasticsearch900Codec() {
+        this(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
+    }
+
+    /**
+     * Constructor. Takes a {@link Zstd814StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of
+     * worse space-efficiency or vice-versa.
+     */
+    public Elasticsearch900Codec(Zstd814StoredFieldsFormat.Mode mode) {
+        super("Elasticsearch900", new Lucene100Codec());
+        this.storedFieldsFormat = mode.getFormat();
+        this.defaultPostingsFormat = new Lucene912PostingsFormat();
+        this.defaultDVFormat = new Lucene90DocValuesFormat();
+        this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public final PostingsFormat postingsFormat() {
+        return postingsFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+
+    @Override
+    public final KnnVectorsFormat knnVectorsFormat() {
+        return knnVectorsFormat;
+    }
+
+    /**
+     * Returns the postings format that should be used for writing new segments of <code>field</code>.
+     *
+     * <p>The default implementation always returns "Lucene912".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation,
+     */
+    public PostingsFormat getPostingsFormatForField(String field) {
+        return defaultPostingsFormat;
+    }
+
+    /**
+     * Returns the docvalues format that should be used for writing new segments of <code>field</code>
+     * .
+     *
+     * <p>The default implementation always returns "Lucene912".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public DocValuesFormat getDocValuesFormatForField(String field) {
+        return defaultDVFormat;
+    }
+
+    /**
+     * Returns the vectors format that should be used for writing new segments of <code>field</code>
+     *
+     * <p>The default implementation always returns "Lucene912".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+        return defaultKnnVectorsFormat;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java
index 64c2ca788f63c..bf2c5a9f01e29 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.mapper.MapperService;
@@ -22,11 +22,11 @@
  * Legacy version of {@link PerFieldMapperCodec}. This codec is preserved to give an escape hatch in case we encounter issues with new
  * changes in {@link PerFieldMapperCodec}.
  */
-public final class LegacyPerFieldMapperCodec extends Lucene912Codec {
+public final class LegacyPerFieldMapperCodec extends Lucene100Codec {
 
     private final PerFieldFormatSupplier formatSupplier;
 
-    public LegacyPerFieldMapperCodec(Lucene912Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
+    public LegacyPerFieldMapperCodec(Lucene100Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
         super(compressionMode);
         this.formatSupplier = new PerFieldFormatSupplier(mapperService, bigArrays);
         // If the below assertion fails, it is a sign that Lucene released a new codec. You must create a copy of the current Elasticsearch
diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
index 83c5cb396d88b..b60b88da5949d 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
@@ -26,7 +26,7 @@
  * per index in real time via the mapping API. If no specific postings format or vector format is
  * configured for a specific field the default postings or vector format is used.
  */
-public final class PerFieldMapperCodec extends Elasticsearch816Codec {
+public final class PerFieldMapperCodec extends Elasticsearch900Codec {
 
     private final PerFieldFormatSupplier formatSupplier;
 
diff --git a/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java
index d26fb52a82bcd..81129835518da 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java
@@ -36,7 +36,6 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.RandomAccessInput;
@@ -142,12 +141,7 @@ static final class FieldsReader extends FieldsProducer {
 
         FieldsReader(SegmentReadState state) throws IOException {
             boolean success = false;
-            try (
-                ChecksumIndexInput metaIn = state.directory.openChecksumInput(
-                    metaFile(state.segmentInfo, state.segmentSuffix),
-                    IOContext.READONCE
-                )
-            ) {
+            try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaFile(state.segmentInfo, state.segmentSuffix))) {
                 CodecUtil.checkIndexHeader(
                     metaIn,
                     BLOOM_CODEC_NAME,
diff --git a/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormat.java
index 01d874adec14d..abf68abe51887 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormat.java
@@ -38,7 +38,6 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.RandomAccessInput;
@@ -291,12 +290,7 @@ static final class FieldsReader extends FieldsProducer {
 
         FieldsReader(SegmentReadState state) throws IOException {
             boolean success = false;
-            try (
-                ChecksumIndexInput metaIn = state.directory.openChecksumInput(
-                    metaFile(state.segmentInfo, state.segmentSuffix),
-                    IOContext.READONCE
-                )
-            ) {
+            try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaFile(state.segmentInfo, state.segmentSuffix))) {
                 Map<String, BloomFilter> bloomFilters = null;
                 Throwable priorE = null;
                 long indexFileLength = 0;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java
index 648913098ff0d..db9c352ee30f8 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java
@@ -21,10 +21,12 @@ public class DocValuesForUtil {
     private static final int BITS_IN_FIVE_BYTES = 5 * Byte.SIZE;
     private static final int BITS_IN_SIX_BYTES = 6 * Byte.SIZE;
     private static final int BITS_IN_SEVEN_BYTES = 7 * Byte.SIZE;
-    private static final int blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
+    private final int blockSize;
     private final byte[] encoded = new byte[1024];
 
-    public DocValuesForUtil() {}
+    public DocValuesForUtil(int numericBlockSize) {
+        this.blockSize = numericBlockSize;
+    }
 
     public static int roundBits(int bitsPerValue) {
         if (bitsPerValue > 24 && bitsPerValue <= 32) {
@@ -67,7 +69,7 @@ private void encodeFiveSixOrSevenBytesPerValue(long[] in, int bitsPerValue, fina
         out.writeBytes(this.encoded, bytesPerValue * in.length);
     }
 
-    public static void decode(int bitsPerValue, final DataInput in, long[] out) throws IOException {
+    public void decode(int bitsPerValue, final DataInput in, long[] out) throws IOException {
         if (bitsPerValue <= 24) {
             ForUtil.decode(bitsPerValue, in, out);
         } else if (bitsPerValue <= 32) {
@@ -81,7 +83,7 @@ public static void decode(int bitsPerValue, final DataInput in, long[] out) thro
         }
     }
 
-    private static void decodeFiveSixOrSevenBytesPerValue(int bitsPerValue, final DataInput in, long[] out) throws IOException {
+    private void decodeFiveSixOrSevenBytesPerValue(int bitsPerValue, final DataInput in, long[] out) throws IOException {
         // NOTE: we expect multibyte values to be written "least significant byte" first
         int bytesPerValue = bitsPerValue / Byte.SIZE;
         long mask = (1L << bitsPerValue) - 1;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java
index 71d9768ac5ff7..dc73428a07c7c 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.codecs.lucene90.IndexedDISI;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipIndexType;
 import org.apache.lucene.index.EmptyDocValuesProducer;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexFileNames;
@@ -41,9 +42,13 @@
 import org.elasticsearch.core.IOUtils;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
+import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SKIP_INDEX_LEVEL_SHIFT;
+import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL;
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SORTED_SET;
 
 final class ES87TSDBDocValuesConsumer extends DocValuesConsumer {
@@ -51,9 +56,16 @@ final class ES87TSDBDocValuesConsumer extends DocValuesConsumer {
     IndexOutput data, meta;
     final int maxDoc;
     private byte[] termsDictBuffer;
-
-    ES87TSDBDocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension)
-        throws IOException {
+    private final int skipIndexIntervalSize;
+
+    ES87TSDBDocValuesConsumer(
+        SegmentWriteState state,
+        int skipIndexIntervalSize,
+        String dataCodec,
+        String dataExtension,
+        String metaCodec,
+        String metaExtension
+    ) throws IOException {
         this.termsDictBuffer = new byte[1 << 14];
         boolean success = false;
         try {
@@ -76,6 +88,7 @@ final class ES87TSDBDocValuesConsumer extends DocValuesConsumer {
                 state.segmentSuffix
             );
             maxDoc = state.segmentInfo.maxDoc();
+            this.skipIndexIntervalSize = skipIndexIntervalSize;
             success = true;
         } finally {
             if (success == false) {
@@ -88,12 +101,17 @@ final class ES87TSDBDocValuesConsumer extends DocValuesConsumer {
     public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
         meta.writeInt(field.number);
         meta.writeByte(ES87TSDBDocValuesFormat.NUMERIC);
-        writeField(field, new EmptyDocValuesProducer() {
+        DocValuesProducer producer = new EmptyDocValuesProducer() {
             @Override
             public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
                 return DocValues.singleton(valuesProducer.getNumeric(field));
             }
-        }, -1);
+        };
+        if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
+            writeSkipIndex(field, producer);
+        }
+
+        writeField(field, producer, -1);
     }
 
     private long[] writeField(FieldInfo field, DocValuesProducer valuesProducer, long maxOrd) throws IOException {
@@ -144,7 +162,7 @@ private long[] writeField(FieldInfo field, DocValuesProducer valuesProducer, lon
             if (maxOrd != 1) {
                 final long[] buffer = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
                 int bufferSize = 0;
-                final ES87TSDBDocValuesEncoder encoder = new ES87TSDBDocValuesEncoder();
+                final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
                 values = valuesProducer.getSortedNumeric(field);
                 final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1;
                 for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
@@ -263,13 +281,11 @@ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) th
     public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
         meta.writeInt(field.number);
         meta.writeByte(ES87TSDBDocValuesFormat.SORTED);
-        doAddSortedField(field, valuesProducer);
+        doAddSortedField(field, valuesProducer, false);
     }
 
-    private void doAddSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
-        SortedDocValues sorted = valuesProducer.getSorted(field);
-        int maxOrd = sorted.getValueCount();
-        writeField(field, new EmptyDocValuesProducer() {
+    private void doAddSortedField(FieldInfo field, DocValuesProducer valuesProducer, boolean addTypeByte) throws IOException {
+        DocValuesProducer producer = new EmptyDocValuesProducer() {
             @Override
             public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
                 SortedDocValues sorted = valuesProducer.getSorted(field);
@@ -306,7 +322,16 @@ public long cost() {
                 };
                 return DocValues.singleton(sortedOrds);
             }
-        }, maxOrd);
+        };
+        if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
+            writeSkipIndex(field, producer);
+        }
+        if (addTypeByte) {
+            meta.writeByte((byte) 0); // multiValued (0 = singleValued)
+        }
+        SortedDocValues sorted = valuesProducer.getSorted(field);
+        int maxOrd = sorted.getValueCount();
+        writeField(field, producer, maxOrd);
         addTermsDict(DocValues.singleton(valuesProducer.getSorted(field)));
     }
 
@@ -459,6 +484,12 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu
     }
 
     private void writeSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer, long maxOrd) throws IOException {
+        if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
+            writeSkipIndex(field, valuesProducer);
+        }
+        if (maxOrd > -1) {
+            meta.writeByte((byte) 1); // multiValued (1 = multiValued)
+        }
         long[] stats = writeField(field, valuesProducer, maxOrd);
         int numDocsWithField = Math.toIntExact(stats[0]);
         long numValues = stats[1];
@@ -510,16 +541,14 @@ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer)
         meta.writeByte(SORTED_SET);
 
         if (isSingleValued(valuesProducer.getSortedSet(field))) {
-            meta.writeByte((byte) 0); // multiValued (0 = singleValued)
             doAddSortedField(field, new EmptyDocValuesProducer() {
                 @Override
                 public SortedDocValues getSorted(FieldInfo field) throws IOException {
                     return SortedSetSelector.wrap(valuesProducer.getSortedSet(field), SortedSetSelector.Type.MIN);
                 }
-            });
+            }, true);
             return;
         }
-        meta.writeByte((byte) 1); // multiValued (1 = multiValued)
 
         SortedSetDocValues values = valuesProducer.getSortedSet(field);
         long maxOrd = values.getValueCount();
@@ -603,4 +632,157 @@ public void close() throws IOException {
             meta = data = null;
         }
     }
+
+    private static class SkipAccumulator {
+        int minDocID;
+        int maxDocID;
+        int docCount;
+        long minValue;
+        long maxValue;
+
+        SkipAccumulator(int docID) {
+            minDocID = docID;
+            minValue = Long.MAX_VALUE;
+            maxValue = Long.MIN_VALUE;
+            docCount = 0;
+        }
+
+        boolean isDone(int skipIndexIntervalSize, int valueCount, long nextValue, int nextDoc) {
+            if (docCount < skipIndexIntervalSize) {
+                return false;
+            }
+            // Once we reach the interval size, we will keep accepting documents if
+            // - next doc value is not a multi-value
+            // - current accumulator only contains a single value and next value is the same value
+            // - the accumulator is dense and the next doc keeps the density (no gaps)
+            return valueCount > 1 || minValue != maxValue || minValue != nextValue || docCount != nextDoc - minDocID;
+        }
+
+        void accumulate(long value) {
+            minValue = Math.min(minValue, value);
+            maxValue = Math.max(maxValue, value);
+        }
+
+        void accumulate(SkipAccumulator other) {
+            assert minDocID <= other.minDocID && maxDocID < other.maxDocID;
+            maxDocID = other.maxDocID;
+            minValue = Math.min(minValue, other.minValue);
+            maxValue = Math.max(maxValue, other.maxValue);
+            docCount += other.docCount;
+        }
+
+        void nextDoc(int docID) {
+            maxDocID = docID;
+            ++docCount;
+        }
+
+        public static SkipAccumulator merge(List<SkipAccumulator> list, int index, int length) {
+            SkipAccumulator acc = new SkipAccumulator(list.get(index).minDocID);
+            for (int i = 0; i < length; i++) {
+                acc.accumulate(list.get(index + i));
+            }
+            return acc;
+        }
+    }
+
+    private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;
+        final long start = data.getFilePointer();
+        final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
+        long globalMaxValue = Long.MIN_VALUE;
+        long globalMinValue = Long.MAX_VALUE;
+        int globalDocCount = 0;
+        int maxDocId = -1;
+        final List<SkipAccumulator> accumulators = new ArrayList<>();
+        SkipAccumulator accumulator = null;
+        final int maxAccumulators = 1 << (SKIP_INDEX_LEVEL_SHIFT * (SKIP_INDEX_MAX_LEVEL - 1));
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            final long firstValue = values.nextValue();
+            if (accumulator != null && accumulator.isDone(skipIndexIntervalSize, values.docValueCount(), firstValue, doc)) {
+                globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
+                globalMinValue = Math.min(globalMinValue, accumulator.minValue);
+                globalDocCount += accumulator.docCount;
+                maxDocId = accumulator.maxDocID;
+                accumulator = null;
+                if (accumulators.size() == maxAccumulators) {
+                    writeLevels(accumulators);
+                    accumulators.clear();
+                }
+            }
+            if (accumulator == null) {
+                accumulator = new SkipAccumulator(doc);
+                accumulators.add(accumulator);
+            }
+            accumulator.nextDoc(doc);
+            accumulator.accumulate(firstValue);
+            for (int i = 1, end = values.docValueCount(); i < end; ++i) {
+                accumulator.accumulate(values.nextValue());
+            }
+        }
+
+        if (accumulators.isEmpty() == false) {
+            globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
+            globalMinValue = Math.min(globalMinValue, accumulator.minValue);
+            globalDocCount += accumulator.docCount;
+            maxDocId = accumulator.maxDocID;
+            writeLevels(accumulators);
+        }
+        meta.writeLong(start); // record the start in meta
+        meta.writeLong(data.getFilePointer() - start); // record the length
+        assert globalDocCount == 0 || globalMaxValue >= globalMinValue;
+        meta.writeLong(globalMaxValue);
+        meta.writeLong(globalMinValue);
+        assert globalDocCount <= maxDocId + 1;
+        meta.writeInt(globalDocCount);
+        meta.writeInt(maxDocId);
+    }
+
+    private void writeLevels(List<SkipAccumulator> accumulators) throws IOException {
+        final List<List<SkipAccumulator>> accumulatorsLevels = new ArrayList<>(SKIP_INDEX_MAX_LEVEL);
+        accumulatorsLevels.add(accumulators);
+        for (int i = 0; i < SKIP_INDEX_MAX_LEVEL - 1; i++) {
+            accumulatorsLevels.add(buildLevel(accumulatorsLevels.get(i)));
+        }
+        int totalAccumulators = accumulators.size();
+        for (int index = 0; index < totalAccumulators; index++) {
+            // compute how many levels we need to write for the current accumulator
+            final int levels = getLevels(index, totalAccumulators);
+            // write the number of levels
+            data.writeByte((byte) levels);
+            // write intervals in reverse order. This is done so we don't
+            // need to read all of them in case of slipping
+            for (int level = levels - 1; level >= 0; level--) {
+                final SkipAccumulator accumulator = accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level));
+                data.writeInt(accumulator.maxDocID);
+                data.writeInt(accumulator.minDocID);
+                data.writeLong(accumulator.maxValue);
+                data.writeLong(accumulator.minValue);
+                data.writeInt(accumulator.docCount);
+            }
+        }
+    }
+
+    private static List<SkipAccumulator> buildLevel(List<SkipAccumulator> accumulators) {
+        final int levelSize = 1 << SKIP_INDEX_LEVEL_SHIFT;
+        final List<SkipAccumulator> collector = new ArrayList<>();
+        for (int i = 0; i < accumulators.size() - levelSize + 1; i += levelSize) {
+            collector.add(SkipAccumulator.merge(accumulators, i, levelSize));
+        }
+        return collector;
+    }
+
+    private static int getLevels(int index, int size) {
+        if (Integer.numberOfTrailingZeros(index) >= SKIP_INDEX_LEVEL_SHIFT) {
+            // TODO: can we do it in constant time rather than linearly with SKIP_INDEX_MAX_LEVEL?
+            final int left = size - index;
+            for (int level = SKIP_INDEX_MAX_LEVEL - 1; level > 0; level--) {
+                final int numberIntervals = 1 << (SKIP_INDEX_LEVEL_SHIFT * level);
+                if (left >= numberIntervals && index % numberIntervals == 0) {
+                    return level + 1;
+                }
+            }
+        }
+        return 1;
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java
index 742249892f61f..496c41b42869a 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java
@@ -43,13 +43,57 @@ public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesF
     static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;
     static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;
 
+    // number of documents in an interval
+    private static final int DEFAULT_SKIP_INDEX_INTERVAL_SIZE = 4096;
+    // bytes on an interval:
+    // * 1 byte : number of levels
+    // * 16 bytes: min / max value,
+    // * 8 bytes: min / max docID
+    // * 4 bytes: number of documents
+    private static final long SKIP_INDEX_INTERVAL_BYTES = 29L;
+    // number of intervals represented as a shift to create a new level, this is 1 << 3 == 8
+    // intervals.
+    static final int SKIP_INDEX_LEVEL_SHIFT = 3;
+    // max number of levels
+    // Increasing this number, it increases how much heap we need at index time.
+    // we currently need (1 * 8 * 8 * 8) = 512 accumulators on heap
+    static final int SKIP_INDEX_MAX_LEVEL = 4;
+    // number of bytes to skip when skipping a level. It does not take into account the
+    // current interval that is being read.
+    static final long[] SKIP_INDEX_JUMP_LENGTH_PER_LEVEL = new long[SKIP_INDEX_MAX_LEVEL];
+
+    static {
+        // Size of the interval minus read bytes (1 byte for level and 4 bytes for maxDocID)
+        SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[0] = SKIP_INDEX_INTERVAL_BYTES - 5L;
+        for (int level = 1; level < SKIP_INDEX_MAX_LEVEL; level++) {
+            // jump from previous level
+            SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] = SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level - 1];
+            // nodes added by new level
+            SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] += (1 << (level * SKIP_INDEX_LEVEL_SHIFT)) * SKIP_INDEX_INTERVAL_BYTES;
+            // remove the byte levels added in the previous level
+            SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] -= (1 << ((level - 1) * SKIP_INDEX_LEVEL_SHIFT));
+        }
+    }
+
+    private final int skipIndexIntervalSize;
+
+    /** Default constructor. */
     public ES87TSDBDocValuesFormat() {
+        this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE);
+    }
+
+    /** Doc values fields format with specified skipIndexIntervalSize. */
+    public ES87TSDBDocValuesFormat(int skipIndexIntervalSize) {
         super(CODEC_NAME);
+        if (skipIndexIntervalSize < 2) {
+            throw new IllegalArgumentException("skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]");
+        }
+        this.skipIndexIntervalSize = skipIndexIntervalSize;
     }
 
     @Override
     public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-        return new ES87TSDBDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+        return new ES87TSDBDocValuesConsumer(state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
index e3c2daddba80e..a7560ce6f3caf 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
@@ -16,6 +16,8 @@
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipIndexType;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.ImpactsEnum;
@@ -27,6 +29,7 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.DataInput;
@@ -43,6 +46,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SKIP_INDEX_JUMP_LENGTH_PER_LEVEL;
+import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL;
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;
 
 public class ES87TSDBDocValuesProducer extends DocValuesProducer {
@@ -51,6 +56,7 @@ public class ES87TSDBDocValuesProducer extends DocValuesProducer {
     private final Map<String, SortedEntry> sorted = new HashMap<>();
     private final Map<String, SortedSetEntry> sortedSets = new HashMap<>();
     private final Map<String, SortedNumericEntry> sortedNumerics = new HashMap<>();
+    private final Map<String, DocValuesSkipperEntry> skippers = new HashMap<>();
     private final IndexInput data;
     private final int maxDoc;
 
@@ -61,7 +67,7 @@ public class ES87TSDBDocValuesProducer extends DocValuesProducer {
 
         // read in the entries from the metadata file.
         int version = -1;
-        try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) {
+        try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName)) {
             Throwable priorE = null;
 
             try {
@@ -126,7 +132,7 @@ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
             return DocValues.emptyBinary();
         }
 
-        final IndexInput bytesSlice = data.slice("fixed-binary", entry.dataOffset, entry.dataLength);
+        final RandomAccessInput bytesSlice = data.randomAccessSlice(entry.dataOffset, entry.dataLength);
 
         if (entry.docsWithFieldOffset == -1) {
             // dense
@@ -138,8 +144,7 @@ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
 
                     @Override
                     public BytesRef binaryValue() throws IOException {
-                        bytesSlice.seek((long) doc * length);
-                        bytesSlice.readBytes(bytes.bytes, 0, length);
+                        bytesSlice.readBytes((long) doc * length, bytes.bytes, 0, length);
                         return bytes;
                     }
                 };
@@ -154,8 +159,7 @@ public BytesRef binaryValue() throws IOException {
                     public BytesRef binaryValue() throws IOException {
                         long startOffset = addresses.get(doc);
                         bytes.length = (int) (addresses.get(doc + 1L) - startOffset);
-                        bytesSlice.seek(startOffset);
-                        bytesSlice.readBytes(bytes.bytes, 0, bytes.length);
+                        bytesSlice.readBytes(startOffset, bytes.bytes, 0, bytes.length);
                         return bytes;
                     }
                 };
@@ -178,8 +182,7 @@ public BytesRef binaryValue() throws IOException {
 
                     @Override
                     public BytesRef binaryValue() throws IOException {
-                        bytesSlice.seek((long) disi.index() * length);
-                        bytesSlice.readBytes(bytes.bytes, 0, length);
+                        bytesSlice.readBytes((long) disi.index() * length, bytes.bytes, 0, length);
                         return bytes;
                     }
                 };
@@ -195,8 +198,7 @@ public BytesRef binaryValue() throws IOException {
                         final int index = disi.index();
                         long startOffset = addresses.get(index);
                         bytes.length = (int) (addresses.get(index + 1L) - startOffset);
-                        bytesSlice.seek(startOffset);
-                        bytesSlice.readBytes(bytes.bytes, 0, bytes.length);
+                        bytesSlice.readBytes(startOffset, bytes.bytes, 0, bytes.length);
                         return bytes;
                     }
                 };
@@ -401,7 +403,7 @@ private static class TermsDict extends BaseTermsEnum {
         final IndexInput bytes;
         final long blockMask;
         final LongValues indexAddresses;
-        final IndexInput indexBytes;
+        final RandomAccessInput indexBytes;
         final BytesRef term;
         long ord = -1;
 
@@ -421,7 +423,7 @@ private static class TermsDict extends BaseTermsEnum {
                 entry.termsIndexAddressesLength
             );
             indexAddresses = DirectMonotonicReader.getInstance(entry.termsIndexAddressesMeta, indexAddressesSlice);
-            indexBytes = data.slice("terms-index", entry.termsIndexOffset, entry.termsIndexLength);
+            indexBytes = data.randomAccessSlice(entry.termsIndexOffset, entry.termsIndexLength);
             term = new BytesRef(entry.maxTermLength);
 
             // add the max term length for the dictionary
@@ -479,8 +481,7 @@ private BytesRef getTermFromIndex(long index) throws IOException {
             assert index >= 0 && index <= (entry.termsDictSize - 1) >>> entry.termsDictIndexShift;
             final long start = indexAddresses.get(index);
             term.length = (int) (indexAddresses.get(index + 1) - start);
-            indexBytes.seek(start);
-            indexBytes.readBytes(term.bytes, 0, term.length);
+            indexBytes.readBytes(start, term.bytes, 0, term.length);
             return term;
         }
 
@@ -659,9 +660,8 @@ public long nextOrd() throws IOException {
                     i = 0;
                     count = ords.docValueCount();
                 }
-                if (i++ == count) {
-                    return NO_MORE_ORDS;
-                }
+                assert i < count;
+                i++;
                 return ords.nextValue();
             }
 
@@ -700,6 +700,116 @@ public long cost() {
         };
     }
 
+    @Override
+    public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
+        final DocValuesSkipperEntry entry = skippers.get(field.name);
+
+        final IndexInput input = data.slice("doc value skipper", entry.offset, entry.length);
+        // Prefetch the first page of data. Following pages are expected to get prefetched through
+        // read-ahead.
+        if (input.length() > 0) {
+            input.prefetch(0, 1);
+        }
+        // TODO: should we write to disk the actual max level for this segment?
+        return new DocValuesSkipper() {
+            final int[] minDocID = new int[SKIP_INDEX_MAX_LEVEL];
+            final int[] maxDocID = new int[SKIP_INDEX_MAX_LEVEL];
+
+            {
+                for (int i = 0; i < SKIP_INDEX_MAX_LEVEL; i++) {
+                    minDocID[i] = maxDocID[i] = -1;
+                }
+            }
+
+            final long[] minValue = new long[SKIP_INDEX_MAX_LEVEL];
+            final long[] maxValue = new long[SKIP_INDEX_MAX_LEVEL];
+            final int[] docCount = new int[SKIP_INDEX_MAX_LEVEL];
+            int levels = 1;
+
+            @Override
+            public void advance(int target) throws IOException {
+                if (target > entry.maxDocId) {
+                    // skipper is exhausted
+                    for (int i = 0; i < SKIP_INDEX_MAX_LEVEL; i++) {
+                        minDocID[i] = maxDocID[i] = DocIdSetIterator.NO_MORE_DOCS;
+                    }
+                } else {
+                    // find next interval
+                    assert target > maxDocID[0] : "target must be bigger that current interval";
+                    while (true) {
+                        levels = input.readByte();
+                        assert levels <= SKIP_INDEX_MAX_LEVEL && levels > 0 : "level out of range [" + levels + "]";
+                        boolean valid = true;
+                        // check if current interval is competitive or we can jump to the next position
+                        for (int level = levels - 1; level >= 0; level--) {
+                            if ((maxDocID[level] = input.readInt()) < target) {
+                                input.skipBytes(SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level]); // the jump for the level
+                                valid = false;
+                                break;
+                            }
+                            minDocID[level] = input.readInt();
+                            maxValue[level] = input.readLong();
+                            minValue[level] = input.readLong();
+                            docCount[level] = input.readInt();
+                        }
+                        if (valid) {
+                            // adjust levels
+                            while (levels < SKIP_INDEX_MAX_LEVEL && maxDocID[levels] >= target) {
+                                levels++;
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+
+            @Override
+            public int numLevels() {
+                return levels;
+            }
+
+            @Override
+            public int minDocID(int level) {
+                return minDocID[level];
+            }
+
+            @Override
+            public int maxDocID(int level) {
+                return maxDocID[level];
+            }
+
+            @Override
+            public long minValue(int level) {
+                return minValue[level];
+            }
+
+            @Override
+            public long maxValue(int level) {
+                return maxValue[level];
+            }
+
+            @Override
+            public int docCount(int level) {
+                return docCount[level];
+            }
+
+            @Override
+            public long minValue() {
+                return entry.minValue;
+            }
+
+            @Override
+            public long maxValue() {
+                return entry.maxValue;
+            }
+
+            @Override
+            public int docCount() {
+                return entry.docCount;
+            }
+        };
+    }
+
     @Override
     public void checkIntegrity() throws IOException {
         CodecUtil.checksumEntireFile(data);
@@ -717,6 +827,9 @@ private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
                 throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
             }
             byte type = meta.readByte();
+            if (info.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
+                skippers.put(info.name, readDocValueSkipperMeta(meta));
+            }
             if (type == ES87TSDBDocValuesFormat.NUMERIC) {
                 numerics.put(info.name, readNumeric(meta));
             } else if (type == ES87TSDBDocValuesFormat.BINARY) {
@@ -739,6 +852,17 @@ private static NumericEntry readNumeric(IndexInput meta) throws IOException {
         return entry;
     }
 
+    private static DocValuesSkipperEntry readDocValueSkipperMeta(IndexInput meta) throws IOException {
+        long offset = meta.readLong();
+        long length = meta.readLong();
+        long maxValue = meta.readLong();
+        long minValue = meta.readLong();
+        int docCount = meta.readInt();
+        int maxDocID = meta.readInt();
+
+        return new DocValuesSkipperEntry(offset, length, minValue, maxValue, docCount, maxDocID);
+    }
+
     private static void readNumeric(IndexInput meta, NumericEntry entry) throws IOException {
         entry.docsWithFieldOffset = meta.readLong();
         entry.docsWithFieldLength = meta.readLong();
@@ -965,7 +1089,7 @@ public long longValue() {
 
                 private final int maxDoc = ES87TSDBDocValuesProducer.this.maxDoc;
                 private int doc = -1;
-                private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder();
+                private final TSDBDocValuesEncoder decoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
                 private long currentBlockIndex = -1;
                 private final long[] currentBlock = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
 
@@ -1030,7 +1154,7 @@ public long longValue() throws IOException {
             );
             return new NumericDocValues() {
 
-                private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder();
+                private final TSDBDocValuesEncoder decoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
                 private long currentBlockIndex = -1;
                 private final long[] currentBlock = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
 
@@ -1092,7 +1216,7 @@ private NumericValues getValues(NumericEntry entry, final long maxOrd) throws IO
         final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1;
         return new NumericValues() {
 
-            private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder();
+            private final TSDBDocValuesEncoder decoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
             private long currentBlockIndex = -1;
             private final long[] currentBlock = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
 
@@ -1249,6 +1373,8 @@ private void set() {
         }
     }
 
+    private record DocValuesSkipperEntry(long offset, long length, long minValue, long maxValue, int docCount, int maxDocId) {}
+
     private static class NumericEntry {
         long docsWithFieldOffset;
         long docsWithFieldLength;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesEncoder.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java
rename to server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesEncoder.java
index 4e95ce34dc410..3af9d726af4fc 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesEncoder.java
@@ -44,8 +44,8 @@
  * </li>
  * </ul>
  *
- * Notice that encoding and decoding are written in a nested way, for instance {@link ES87TSDBDocValuesEncoder#deltaEncode} calling
- * {@link ES87TSDBDocValuesEncoder#removeOffset} and so on. This allows us to easily introduce new encoding schemes or remove existing
+ * Notice that encoding and decoding are written in a nested way, for instance {@link TSDBDocValuesEncoder#deltaEncode} calling
+ * {@link TSDBDocValuesEncoder#removeOffset} and so on. This allows us to easily introduce new encoding schemes or remove existing
  * (non-effective) encoding schemes in a backward-compatible way.
  *
  * A token is used as a bitmask to represent which encoding is applied and allows us to detect the applied encoding scheme at decoding time.
@@ -54,11 +54,13 @@
  *
  * Of course, decoding follows the opposite order with respect to encoding.
  */
-public class ES87TSDBDocValuesEncoder {
+public class TSDBDocValuesEncoder {
     private final DocValuesForUtil forUtil;
+    private final int numericBlockSize;
 
-    public ES87TSDBDocValuesEncoder() {
-        this.forUtil = new DocValuesForUtil();
+    public TSDBDocValuesEncoder(int numericBlockSize) {
+        this.forUtil = new DocValuesForUtil(numericBlockSize);
+        this.numericBlockSize = numericBlockSize;
     }
 
     /**
@@ -68,7 +70,7 @@ public ES87TSDBDocValuesEncoder() {
     private void deltaEncode(int token, int tokenBits, long[] in, DataOutput out) throws IOException {
         int gts = 0;
         int lts = 0;
-        for (int i = 1; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
+        for (int i = 1; i < numericBlockSize; ++i) {
             if (in[i] > in[i - 1]) {
                 gts++;
             } else if (in[i] < in[i - 1]) {
@@ -79,7 +81,7 @@ private void deltaEncode(int token, int tokenBits, long[] in, DataOutput out) th
         final boolean doDeltaCompression = (gts == 0 && lts >= 2) || (lts == 0 && gts >= 2);
         long first = 0;
         if (doDeltaCompression) {
-            for (int i = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE - 1; i > 0; --i) {
+            for (int i = numericBlockSize - 1; i > 0; --i) {
                 in[i] -= in[i - 1];
             }
             // Avoid setting in[0] to 0 in case there is a minimum interval between
@@ -115,7 +117,7 @@ private void removeOffset(int token, int tokenBits, long[] in, DataOutput out) t
         }
 
         if (min != 0) {
-            for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
+            for (int i = 0; i < numericBlockSize; ++i) {
                 in[i] -= min;
             }
             token = (token << 1) | 0x01;
@@ -143,7 +145,7 @@ private void gcdEncode(int token, int tokenBits, long[] in, DataOutput out) thro
         }
         final boolean doGcdCompression = Long.compareUnsigned(gcd, 1) > 0;
         if (doGcdCompression) {
-            for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
+            for (int i = 0; i < numericBlockSize; ++i) {
                 in[i] /= gcd;
             }
             token = (token << 1) | 0x01;
@@ -174,7 +176,7 @@ private void forEncode(int token, int tokenBits, long[] in, DataOutput out) thro
      * Encode the given longs using a combination of delta-coding, GCD factorization and bit packing.
      */
     void encode(long[] in, DataOutput out) throws IOException {
-        assert in.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
+        assert in.length == numericBlockSize;
 
         deltaEncode(0, 0, in, out);
     }
@@ -192,7 +194,7 @@ void encode(long[] in, DataOutput out) throws IOException {
      * </ul>
      */
     void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
-        assert in.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
+        assert in.length == numericBlockSize;
         int numRuns = 1;
         long firstValue = in[0];
         long previousValue = firstValue;
@@ -259,7 +261,7 @@ void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOExceptio
     }
 
     void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
-        assert out.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE : out.length;
+        assert out.length == numericBlockSize : out.length;
 
         long v1 = in.readVLong();
         int encoding = Long.numberOfTrailingZeros(~v1);
@@ -275,7 +277,7 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException
             Arrays.fill(out, runLen, out.length, v2);
         } else if (encoding == 2) {
             // bit-packed
-            DocValuesForUtil.decode(bitsPerOrd, in, out);
+            forUtil.decode(bitsPerOrd, in, out);
         } else if (encoding == 3) {
             // cycle encoding
             int cycleLength = (int) v1;
@@ -293,13 +295,13 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException
 
     /** Decode longs that have been encoded with {@link #encode}. */
     void decode(DataInput in, long[] out) throws IOException {
-        assert out.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE : out.length;
+        assert out.length == numericBlockSize : out.length;
 
         final int token = in.readVInt();
         final int bitsPerValue = token >>> 3;
 
         if (bitsPerValue != 0) {
-            DocValuesForUtil.decode(bitsPerValue, in, out);
+            forUtil.decode(bitsPerValue, in, out);
         } else {
             Arrays.fill(out, 0L);
         }
@@ -330,21 +332,21 @@ void decode(DataInput in, long[] out) throws IOException {
     }
 
     // this loop should auto-vectorize
-    private static void mul(long[] arr, long m) {
-        for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
+    private void mul(long[] arr, long m) {
+        for (int i = 0; i < numericBlockSize; ++i) {
             arr[i] *= m;
         }
     }
 
     // this loop should auto-vectorize
-    private static void add(long[] arr, long min) {
-        for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
+    private void add(long[] arr, long min) {
+        for (int i = 0; i < numericBlockSize; ++i) {
             arr[i] += min;
         }
     }
 
-    private static void deltaDecode(long[] arr) {
-        for (int i = 1; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
+    private void deltaDecode(long[] arr) {
+        for (int i = 1; i < numericBlockSize; ++i) {
             arr[i] += arr[i - 1];
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java
index 3d2acb533e26d..5201e57179cc7 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java
@@ -27,6 +27,14 @@
 public class BQVectorUtils {
     private static final float EPSILON = 1e-4f;
 
+    public static double sqrtNewtonRaphson(double x, double curr, double prev) {
+        return (curr == prev) ? curr : sqrtNewtonRaphson(x, 0.5 * (curr + x / curr), curr);
+    }
+
+    public static double constSqrt(double x) {
+        return x >= 0 && Double.isInfinite(x) == false ? sqrtNewtonRaphson(x, x, 0) : Double.NaN;
+    }
+
     public static boolean isUnitVector(float[] v) {
         double l1norm = VectorUtil.dotProduct(v, v);
         return Math.abs(l1norm - 1.0d) <= EPSILON;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java
index 73dd4273a794e..cf69ab0862949 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java
@@ -19,23 +19,52 @@
  */
 package org.elasticsearch.index.codec.vectors;
 
-import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.util.VectorUtil;
 
 import java.io.IOException;
 
+import static org.elasticsearch.index.codec.vectors.BQVectorUtils.constSqrt;
+
 /**
  * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10
  */
-public abstract class BinarizedByteVectorValues extends DocIdSetIterator {
-
-    public abstract float[] getCorrectiveTerms();
+public abstract class BinarizedByteVectorValues extends ByteVectorValues {
 
-    public abstract byte[] vectorValue() throws IOException;
+    public abstract float[] getCorrectiveTerms(int vectorOrd) throws IOException;
 
     /** Return the dimension of the vectors */
     public abstract int dimension();
 
+    /** Returns the centroid distance for the vector */
+    public abstract float getCentroidDistance(int vectorOrd) throws IOException;
+
+    /** Returns the vector magnitude for the vector */
+    public abstract float getVectorMagnitude(int vectorOrd) throws IOException;
+
+    /** Returns OOQ corrective factor for the given vector ordinal */
+    public abstract float getOOQ(int targetOrd) throws IOException;
+
+    /**
+     * Returns the norm of the target vector w the centroid corrective factor for the given vector
+     * ordinal
+     */
+    public abstract float getNormOC(int targetOrd) throws IOException;
+
+    /**
+     * Returns the target vector dot product the centroid corrective factor for the given vector
+     * ordinal
+     */
+    public abstract float getODotC(int targetOrd) throws IOException;
+
+    /**
+     * @return the quantizer used to quantize the vectors
+     */
+    public abstract BinaryQuantizer getQuantizer();
+
+    public abstract float[] getCentroid() throws IOException;
+
     /**
      * Return the number of vectors for this field.
      *
@@ -43,9 +72,16 @@ public abstract class BinarizedByteVectorValues extends DocIdSetIterator {
      */
     public abstract int size();
 
-    @Override
-    public final long cost() {
-        return size();
+    int discretizedDimensions() {
+        return BQVectorUtils.discretize(dimension(), 64);
+    }
+
+    float sqrtDimensions() {
+        return (float) constSqrt(dimension());
+    }
+
+    float maxX1() {
+        return (float) (1.9 / constSqrt(discretizedDimensions() - 1.0));
     }
 
     /**
@@ -55,4 +91,13 @@ public final long cost() {
      * @return a {@link VectorScorer} instance or null
      */
     public abstract VectorScorer scorer(float[] query) throws IOException;
+
+    @Override
+    public abstract BinarizedByteVectorValues copy() throws IOException;
+
+    float getCentroidDP() throws IOException {
+        // this only gets executed on-merge
+        float[] centroid = getCentroid();
+        return VectorUtil.dotProduct(centroid, centroid);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java
index cc5454ee074e6..ab882c8b04648 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java
@@ -152,10 +152,5 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits
         public void close() throws IOException {
             reader.close();
         }
-
-        @Override
-        public long ramBytesUsed() {
-            return reader.ramBytesUsed();
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
index 9491598653c44..662e4040511e2 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
@@ -160,11 +160,5 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits
         public void close() throws IOException {
             reader.close();
         }
-
-        @Override
-        public long ramBytesUsed() {
-            return reader.ramBytesUsed();
-        }
-
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
index 4bf396e8d5ad1..4c4fd00806954 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
@@ -22,18 +22,17 @@
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.MergeState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Sorter;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;
-import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.QuantizedVectorsReader;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 import org.elasticsearch.simdvec.VectorScorerFactory;
 import org.elasticsearch.simdvec.VectorSimilarityType;
@@ -41,6 +40,7 @@
 import java.io.IOException;
 
 import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL;
+import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT;
 
 public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
 
@@ -245,9 +245,9 @@ public String toString() {
         }
 
         @Override
-        public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction sim, RandomAccessVectorValues values)
+        public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction sim, KnnVectorValues values)
             throws IOException {
-            if (values instanceof RandomAccessQuantizedByteVectorValues qValues && values.getSlice() != null) {
+            if (values instanceof QuantizedByteVectorValues qValues && qValues.getSlice() != null) {
                 // TODO: optimize int4 quantization
                 if (qValues.getScalarQuantizer().getBits() != 7) {
                     return delegate.getRandomVectorScorerSupplier(sim, values);
@@ -255,7 +255,7 @@ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarity
                 if (factory != null) {
                     var scorer = factory.getInt7SQVectorScorerSupplier(
                         VectorSimilarityType.of(sim),
-                        values.getSlice(),
+                        qValues.getSlice(),
                         qValues,
                         qValues.getScalarQuantizer().getConstantMultiplier()
                     );
@@ -268,9 +268,9 @@ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarity
         }
 
         @Override
-        public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, RandomAccessVectorValues values, float[] query)
+        public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, KnnVectorValues values, float[] query)
             throws IOException {
-            if (values instanceof RandomAccessQuantizedByteVectorValues qValues && values.getSlice() != null) {
+            if (values instanceof QuantizedByteVectorValues qValues && qValues.getSlice() != null) {
                 // TODO: optimize int4 quantization
                 if (qValues.getScalarQuantizer().getBits() != 7) {
                     return delegate.getRandomVectorScorer(sim, values, query);
@@ -286,9 +286,14 @@ public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, Ra
         }
 
         @Override
-        public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, RandomAccessVectorValues values, byte[] query)
+        public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, KnnVectorValues values, byte[] query)
             throws IOException {
             return delegate.getRandomVectorScorer(sim, values, query);
         }
     }
+
+    @Override
+    public int getMaxDimensions(String fieldName) {
+        return MAX_DIMS_COUNT;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java
index f0f25bd702749..18668f4f304b0 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java
@@ -14,17 +14,20 @@
 import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
 import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
 import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
+import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.util.VectorUtil;
-import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 
 import java.io.IOException;
 
+import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT;
+
 class ES815BitFlatVectorsFormat extends FlatVectorsFormat {
 
     private static final FlatVectorsFormat delegate = new Lucene99FlatVectorsFormat(FlatBitVectorScorer.INSTANCE);
@@ -43,6 +46,11 @@ public FlatVectorsReader fieldsReader(SegmentReadState segmentReadState) throws
         return delegate.fieldsReader(segmentReadState);
     }
 
+    @Override
+    public int getMaxDimensions(String fieldName) {
+        return MAX_DIMS_COUNT;
+    }
+
     static class FlatBitVectorScorer implements FlatVectorsScorer {
 
         static final FlatBitVectorScorer INSTANCE = new FlatBitVectorScorer();
@@ -61,14 +69,14 @@ public String toString() {
         @Override
         public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
             VectorSimilarityFunction vectorSimilarityFunction,
-            RandomAccessVectorValues randomAccessVectorValues
+            KnnVectorValues vectorValues
         ) throws IOException {
-            assert randomAccessVectorValues instanceof RandomAccessVectorValues.Bytes;
+            assert vectorValues instanceof ByteVectorValues;
             assert vectorSimilarityFunction == VectorSimilarityFunction.EUCLIDEAN;
-            if (randomAccessVectorValues instanceof RandomAccessVectorValues.Bytes randomAccessVectorValuesBytes) {
-                assert randomAccessVectorValues instanceof RandomAccessQuantizedByteVectorValues == false;
+            if (vectorValues instanceof ByteVectorValues byteVectorValues) {
+                assert byteVectorValues instanceof QuantizedByteVectorValues == false;
                 return switch (vectorSimilarityFunction) {
-                    case DOT_PRODUCT, MAXIMUM_INNER_PRODUCT, COSINE, EUCLIDEAN -> new HammingScorerSupplier(randomAccessVectorValuesBytes);
+                    case DOT_PRODUCT, MAXIMUM_INNER_PRODUCT, COSINE, EUCLIDEAN -> new HammingScorerSupplier(byteVectorValues);
                 };
             }
             throw new IllegalArgumentException("Unsupported vector type or similarity function");
@@ -77,18 +85,15 @@ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
         @Override
         public RandomVectorScorer getRandomVectorScorer(
             VectorSimilarityFunction vectorSimilarityFunction,
-            RandomAccessVectorValues randomAccessVectorValues,
-            byte[] bytes
-        ) {
-            assert randomAccessVectorValues instanceof RandomAccessVectorValues.Bytes;
+            KnnVectorValues vectorValues,
+            byte[] target
+        ) throws IOException {
+            assert vectorValues instanceof ByteVectorValues;
             assert vectorSimilarityFunction == VectorSimilarityFunction.EUCLIDEAN;
-            if (randomAccessVectorValues instanceof RandomAccessVectorValues.Bytes randomAccessVectorValuesBytes) {
-                checkDimensions(bytes.length, randomAccessVectorValuesBytes.dimension());
+            if (vectorValues instanceof ByteVectorValues byteVectorValues) {
+                checkDimensions(target.length, byteVectorValues.dimension());
                 return switch (vectorSimilarityFunction) {
-                    case DOT_PRODUCT, MAXIMUM_INNER_PRODUCT, COSINE, EUCLIDEAN -> new HammingVectorScorer(
-                        randomAccessVectorValuesBytes,
-                        bytes
-                    );
+                    case DOT_PRODUCT, MAXIMUM_INNER_PRODUCT, COSINE, EUCLIDEAN -> new HammingVectorScorer(byteVectorValues, target);
                 };
             }
             throw new IllegalArgumentException("Unsupported vector type or similarity function");
@@ -96,10 +101,10 @@ public RandomVectorScorer getRandomVectorScorer(
 
         @Override
         public RandomVectorScorer getRandomVectorScorer(
-            VectorSimilarityFunction vectorSimilarityFunction,
-            RandomAccessVectorValues randomAccessVectorValues,
-            float[] floats
-        ) {
+            VectorSimilarityFunction similarityFunction,
+            KnnVectorValues vectorValues,
+            float[] target
+        ) throws IOException {
             throw new IllegalArgumentException("Unsupported vector type");
         }
     }
@@ -110,9 +115,9 @@ static float hammingScore(byte[] a, byte[] b) {
 
     static class HammingVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer {
         private final byte[] query;
-        private final RandomAccessVectorValues.Bytes byteValues;
+        private final ByteVectorValues byteValues;
 
-        HammingVectorScorer(RandomAccessVectorValues.Bytes byteValues, byte[] query) {
+        HammingVectorScorer(ByteVectorValues byteValues, byte[] query) {
             super(byteValues);
             this.query = query;
             this.byteValues = byteValues;
@@ -125,9 +130,9 @@ public float score(int i) throws IOException {
     }
 
     static class HammingScorerSupplier implements RandomVectorScorerSupplier {
-        private final RandomAccessVectorValues.Bytes byteValues, byteValues1, byteValues2;
+        private final ByteVectorValues byteValues, byteValues1, byteValues2;
 
-        HammingScorerSupplier(RandomAccessVectorValues.Bytes byteValues) throws IOException {
+        HammingScorerSupplier(ByteVectorValues byteValues) throws IOException {
             this.byteValues = byteValues;
             this.byteValues1 = byteValues.copy();
             this.byteValues2 = byteValues.copy();
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java
index 78fa282709098..72c5da4880e75 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.index.codec.vectors;
 
 import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.VectorUtil;
-import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
 import org.elasticsearch.simdvec.ESVectorUtil;
@@ -45,9 +45,9 @@ public ES816BinaryFlatVectorsScorer(FlatVectorsScorer nonQuantizedDelegate) {
     @Override
     public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
         VectorSimilarityFunction similarityFunction,
-        RandomAccessVectorValues vectorValues
+        KnnVectorValues vectorValues
     ) throws IOException {
-        if (vectorValues instanceof RandomAccessBinarizedByteVectorValues) {
+        if (vectorValues instanceof BinarizedByteVectorValues) {
             throw new UnsupportedOperationException(
                 "getRandomVectorScorerSupplier(VectorSimilarityFunction,RandomAccessVectorValues) not implemented for binarized format"
             );
@@ -58,10 +58,10 @@ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
     @Override
     public RandomVectorScorer getRandomVectorScorer(
         VectorSimilarityFunction similarityFunction,
-        RandomAccessVectorValues vectorValues,
+        KnnVectorValues vectorValues,
         float[] target
     ) throws IOException {
-        if (vectorValues instanceof RandomAccessBinarizedByteVectorValues binarizedVectors) {
+        if (vectorValues instanceof BinarizedByteVectorValues binarizedVectors) {
             BinaryQuantizer quantizer = binarizedVectors.getQuantizer();
             float[] centroid = binarizedVectors.getCentroid();
             // FIXME: precompute this once?
@@ -82,7 +82,7 @@ public RandomVectorScorer getRandomVectorScorer(
     @Override
     public RandomVectorScorer getRandomVectorScorer(
         VectorSimilarityFunction similarityFunction,
-        RandomAccessVectorValues vectorValues,
+        KnnVectorValues vectorValues,
         byte[] target
     ) throws IOException {
         return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
@@ -91,7 +91,7 @@ public RandomVectorScorer getRandomVectorScorer(
     RandomVectorScorerSupplier getRandomVectorScorerSupplier(
         VectorSimilarityFunction similarityFunction,
         ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues scoringVectors,
-        RandomAccessBinarizedByteVectorValues targetVectors
+        BinarizedByteVectorValues targetVectors
     ) {
         return new BinarizedRandomVectorScorerSupplier(scoringVectors, targetVectors, similarityFunction);
     }
@@ -104,12 +104,12 @@ public String toString() {
     /** Vector scorer supplier over binarized vector values */
     static class BinarizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier {
         private final ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors;
-        private final RandomAccessBinarizedByteVectorValues targetVectors;
+        private final BinarizedByteVectorValues targetVectors;
         private final VectorSimilarityFunction similarityFunction;
 
         BinarizedRandomVectorScorerSupplier(
             ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors,
-            RandomAccessBinarizedByteVectorValues targetVectors,
+            BinarizedByteVectorValues targetVectors,
             VectorSimilarityFunction similarityFunction
         ) {
             this.queryVectors = queryVectors;
@@ -149,14 +149,15 @@ public record BinaryQueryVector(byte[] vector, BinaryQuantizer.QueryFactors fact
     /** Vector scorer over binarized vector values */
     public static class BinarizedRandomVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer {
         private final BinaryQueryVector queryVector;
-        private final RandomAccessBinarizedByteVectorValues targetVectors;
+        private final BinarizedByteVectorValues targetVectors;
         private final VectorSimilarityFunction similarityFunction;
 
         private final float sqrtDimensions;
+        private final float maxX1;
 
         public BinarizedRandomVectorScorer(
             BinaryQueryVector queryVectors,
-            RandomAccessBinarizedByteVectorValues targetVectors,
+            BinarizedByteVectorValues targetVectors,
             VectorSimilarityFunction similarityFunction
         ) {
             super(targetVectors);
@@ -164,24 +165,12 @@ public BinarizedRandomVectorScorer(
             this.targetVectors = targetVectors;
             this.similarityFunction = similarityFunction;
             // FIXME: precompute this once?
-            this.sqrtDimensions = (float) Utils.constSqrt(targetVectors.dimension());
-        }
-
-        // FIXME: utils class; pull this out
-        private static class Utils {
-            public static double sqrtNewtonRaphson(double x, double curr, double prev) {
-                return (curr == prev) ? curr : sqrtNewtonRaphson(x, 0.5 * (curr + x / curr), curr);
-            }
-
-            public static double constSqrt(double x) {
-                return x >= 0 && Double.isInfinite(x) == false ? sqrtNewtonRaphson(x, x, 0) : Double.NaN;
-            }
+            this.sqrtDimensions = targetVectors.sqrtDimensions();
+            this.maxX1 = targetVectors.maxX1();
         }
 
         @Override
         public float score(int targetOrd) throws IOException {
-            // FIXME: implement fastscan in the future?
-
             byte[] quantizedQuery = queryVector.vector();
             int quantizedSum = queryVector.factors().quantizedSum();
             float lower = queryVector.factors().lower();
@@ -218,17 +207,13 @@ public float score(int targetOrd) throws IOException {
             }
             assert Float.isFinite(dist);
 
-            // TODO: this is useful for mandatory rescoring by accounting for bias
-            // However, for just oversampling & rescoring, it isn't strictly useful.
-            // We should consider utilizing this bias in the future to determine which vectors need to
-            // be rescored
-            // float ooqSqr = (float) Math.pow(ooq, 2);
-            // float errorBound = (float) (normVmC * normOC * (maxX1 * Math.sqrt((1 - ooqSqr) / ooqSqr)));
-            // float score = dist - errorBound;
+            float ooqSqr = (float) Math.pow(ooq, 2);
+            float errorBound = (float) (vmC * normOC * (maxX1 * Math.sqrt((1 - ooqSqr) / ooqSqr)));
+            float score = Float.isFinite(errorBound) ? dist - errorBound : dist;
             if (similarityFunction == MAXIMUM_INNER_PRODUCT) {
-                return VectorUtil.scaleMaxInnerProductScore(dist);
+                return VectorUtil.scaleMaxInnerProductScore(score);
             }
-            return Math.max((1f + dist) / 2f, 0);
+            return Math.max((1f + score) / 2f, 0);
         }
 
         private float euclideanScore(
@@ -256,17 +241,13 @@ private float euclideanScore(
 
             long qcDist = ESVectorUtil.ipByteBinByte(quantizedQuery, binaryCode);
             float score = sqrX + distanceToCentroid + factorPPC * lower + (qcDist * 2 - quantizedSum) * factorIP * width;
-            // TODO: this is useful for mandatory rescoring by accounting for bias
-            // However, for just oversampling & rescoring, it isn't strictly useful.
-            // We should consider utilizing this bias in the future to determine which vectors need to
-            // be rescored
-            // float projectionDist = (float) Math.sqrt(xX0 * xX0 - targetDistToC * targetDistToC);
-            // float error = 2.0f * maxX1 * projectionDist;
-            // float y = (float) Math.sqrt(distanceToCentroid);
-            // float errorBound = y * error;
-            // if (Float.isFinite(errorBound)) {
-            // score = dist + errorBound;
-            // }
+            float projectionDist = (float) Math.sqrt(xX0 * xX0 - targetDistToC * targetDistToC);
+            float error = 2.0f * maxX1 * projectionDist;
+            float y = (float) Math.sqrt(distanceToCentroid);
+            float errorBound = y * error;
+            if (Float.isFinite(errorBound)) {
+                score = score + errorBound;
+            }
             return Math.max(1 / (1f + score), 0);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java
index b0378fee6793d..21c4a5c449387 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.ReadAdvice;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.RamUsageEstimator;
@@ -78,7 +79,7 @@ public ES816BinaryQuantizedVectorsReader(
             ES816BinaryQuantizedVectorsFormat.META_EXTENSION
         );
         boolean success = false;
-        try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName, state.context)) {
+        try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
             Throwable priorE = null;
             try {
                 versionMeta = CodecUtil.checkIndexHeader(
@@ -102,7 +103,7 @@ public ES816BinaryQuantizedVectorsReader(
                 ES816BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,
                 // Quantized vectors are accessed randomly from their node ID stored in the HNSW
                 // graph.
-                state.context.withRandomAccess()
+                state.context.withReadAdvice(ReadAdvice.RANDOM)
             );
             success = true;
         } finally {
@@ -357,9 +358,9 @@ static FieldEntry create(IndexInput input, VectorEncoding vectorEncoding, Vector
     /** Binarized vector values holding row and quantized vector values */
     protected static final class BinarizedVectorValues extends FloatVectorValues {
         private final FloatVectorValues rawVectorValues;
-        private final OffHeapBinarizedVectorValues quantizedVectorValues;
+        private final BinarizedByteVectorValues quantizedVectorValues;
 
-        BinarizedVectorValues(FloatVectorValues rawVectorValues, OffHeapBinarizedVectorValues quantizedVectorValues) {
+        BinarizedVectorValues(FloatVectorValues rawVectorValues, BinarizedByteVectorValues quantizedVectorValues) {
             this.rawVectorValues = rawVectorValues;
             this.quantizedVectorValues = quantizedVectorValues;
         }
@@ -375,29 +376,28 @@ public int size() {
         }
 
         @Override
-        public float[] vectorValue() throws IOException {
-            return rawVectorValues.vectorValue();
+        public float[] vectorValue(int ord) throws IOException {
+            return rawVectorValues.vectorValue(ord);
         }
 
         @Override
-        public int docID() {
-            return rawVectorValues.docID();
+        public BinarizedVectorValues copy() throws IOException {
+            return new BinarizedVectorValues(rawVectorValues.copy(), quantizedVectorValues.copy());
         }
 
         @Override
-        public int nextDoc() throws IOException {
-            int rawDocId = rawVectorValues.nextDoc();
-            int quantizedDocId = quantizedVectorValues.nextDoc();
-            assert rawDocId == quantizedDocId;
-            return quantizedDocId;
+        public Bits getAcceptOrds(Bits acceptDocs) {
+            return rawVectorValues.getAcceptOrds(acceptDocs);
         }
 
         @Override
-        public int advance(int target) throws IOException {
-            int rawDocId = rawVectorValues.advance(target);
-            int quantizedDocId = quantizedVectorValues.advance(target);
-            assert rawDocId == quantizedDocId;
-            return quantizedDocId;
+        public int ordToDoc(int ord) {
+            return rawVectorValues.ordToDoc(ord);
+        }
+
+        @Override
+        public DocIndexIterator iterator() {
+            return rawVectorValues.iterator();
         }
 
         @Override
@@ -405,7 +405,7 @@ public VectorScorer scorer(float[] query) throws IOException {
             return quantizedVectorValues.scorer(query);
         }
 
-        protected OffHeapBinarizedVectorValues getQuantizedVectorValues() throws IOException {
+        protected BinarizedByteVectorValues getQuantizedVectorValues() throws IOException {
             return quantizedVectorValues;
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java
index 92837a8ffce45..a7774b850b64c 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java
@@ -30,6 +30,7 @@
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.MergeState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Sorter;
@@ -44,7 +45,6 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;
-import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
 import org.elasticsearch.core.SuppressForbidden;
@@ -354,10 +354,11 @@ static DocsWithFieldSet writeBinarizedVectorAndQueryData(
         int queryCorrectionCount = binaryQuantizer.getSimilarity() != EUCLIDEAN ? 5 : 3;
         final ByteBuffer queryCorrectionsBuffer = ByteBuffer.allocate(Float.BYTES * queryCorrectionCount + Short.BYTES)
             .order(ByteOrder.LITTLE_ENDIAN);
-        for (int docV = floatVectorValues.nextDoc(); docV != NO_MORE_DOCS; docV = floatVectorValues.nextDoc()) {
+        KnnVectorValues.DocIndexIterator iterator = floatVectorValues.iterator();
+        for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) {
             // write index vector
             BinaryQuantizer.QueryAndIndexResults r = binaryQuantizer.quantizeQueryAndIndex(
-                floatVectorValues.vectorValue(),
+                floatVectorValues.vectorValue(iterator.index()),
                 toIndex,
                 toQuery,
                 centroid
@@ -393,11 +394,12 @@ static DocsWithFieldSet writeBinarizedVectorAndQueryData(
     static DocsWithFieldSet writeBinarizedVectorData(IndexOutput output, BinarizedByteVectorValues binarizedByteVectorValues)
         throws IOException {
         DocsWithFieldSet docsWithField = new DocsWithFieldSet();
-        for (int docV = binarizedByteVectorValues.nextDoc(); docV != NO_MORE_DOCS; docV = binarizedByteVectorValues.nextDoc()) {
+        KnnVectorValues.DocIndexIterator iterator = binarizedByteVectorValues.iterator();
+        for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) {
             // write vector
-            byte[] binaryValue = binarizedByteVectorValues.vectorValue();
+            byte[] binaryValue = binarizedByteVectorValues.vectorValue(iterator.index());
             output.writeBytes(binaryValue, binaryValue.length);
-            float[] corrections = binarizedByteVectorValues.getCorrectiveTerms();
+            float[] corrections = binarizedByteVectorValues.getCorrectiveTerms(iterator.index());
             for (int i = 0; i < corrections.length; i++) {
                 output.writeInt(Float.floatToIntBits(corrections[i]));
             }
@@ -598,8 +600,9 @@ static int calculateCentroid(MergeState mergeState, FieldInfo fieldInfo, float[]
             if (vectorValues == null) {
                 continue;
             }
-            for (int doc = vectorValues.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = vectorValues.nextDoc()) {
-                float[] vector = vectorValues.vectorValue();
+            KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
+            for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
+                float[] vector = vectorValues.vectorValue(iterator.index());
                 // TODO Panama sum
                 for (int j = 0; j < vector.length; j++) {
                     centroid[j] += vector[j];
@@ -827,23 +830,31 @@ static class BinarizedFloatVectorValues extends BinarizedByteVectorValues {
         private final float[] centroid;
         private final FloatVectorValues values;
         private final BinaryQuantizer quantizer;
-        private int lastDoc;
+        private int lastOrd = -1;
 
         BinarizedFloatVectorValues(FloatVectorValues delegate, BinaryQuantizer quantizer, float[] centroid) {
             this.values = delegate;
             this.quantizer = quantizer;
             this.binarized = new byte[BQVectorUtils.discretize(delegate.dimension(), 64) / 8];
             this.centroid = centroid;
-            lastDoc = -1;
         }
 
         @Override
-        public float[] getCorrectiveTerms() {
+        public float[] getCorrectiveTerms(int ord) {
+            if (ord != lastOrd) {
+                throw new IllegalStateException(
+                    "attempt to retrieve corrective terms for different ord " + ord + " than the quantization was done for: " + lastOrd
+                );
+            }
             return corrections;
         }
 
         @Override
-        public byte[] vectorValue() throws IOException {
+        public byte[] vectorValue(int ord) throws IOException {
+            if (ord != lastOrd) {
+                binarize(ord);
+                lastOrd = ord;
+            }
             return binarized;
         }
 
@@ -853,33 +864,43 @@ public int dimension() {
         }
 
         @Override
-        public int size() {
-            return values.size();
+        public float getCentroidDistance(int vectorOrd) throws IOException {
+            throw new UnsupportedOperationException();
         }
 
         @Override
-        public int docID() {
-            return values.docID();
+        public float getVectorMagnitude(int vectorOrd) throws IOException {
+            throw new UnsupportedOperationException();
         }
 
         @Override
-        public int nextDoc() throws IOException {
-            int doc = values.nextDoc();
-            if (doc != NO_MORE_DOCS) {
-                binarize();
-            }
-            lastDoc = doc;
-            return doc;
+        public float getOOQ(int targetOrd) throws IOException {
+            throw new UnsupportedOperationException();
         }
 
         @Override
-        public int advance(int target) throws IOException {
-            int doc = values.advance(target);
-            if (doc != NO_MORE_DOCS) {
-                binarize();
-            }
-            lastDoc = doc;
-            return doc;
+        public float getNormOC(int targetOrd) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public float getODotC(int targetOrd) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public BinaryQuantizer getQuantizer() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public float[] getCentroid() throws IOException {
+            return centroid;
+        }
+
+        @Override
+        public int size() {
+            return values.size();
         }
 
         @Override
@@ -887,22 +908,32 @@ public VectorScorer scorer(float[] target) throws IOException {
             throw new UnsupportedOperationException();
         }
 
-        private void binarize() throws IOException {
-            if (lastDoc == docID()) return;
-            corrections = quantizer.quantizeForIndex(values.vectorValue(), binarized, centroid);
+        @Override
+        public BinarizedByteVectorValues copy() throws IOException {
+            return new BinarizedFloatVectorValues(values.copy(), quantizer, centroid);
+        }
+
+        private void binarize(int ord) throws IOException {
+            corrections = quantizer.quantizeForIndex(values.vectorValue(ord), binarized, centroid);
+        }
+
+        @Override
+        public DocIndexIterator iterator() {
+            return values.iterator();
+        }
+
+        @Override
+        public int ordToDoc(int ord) {
+            return values.ordToDoc(ord);
         }
     }
 
     static class BinarizedCloseableRandomVectorScorerSupplier implements CloseableRandomVectorScorerSupplier {
         private final RandomVectorScorerSupplier supplier;
-        private final RandomAccessVectorValues vectorValues;
+        private final KnnVectorValues vectorValues;
         private final Closeable onClose;
 
-        BinarizedCloseableRandomVectorScorerSupplier(
-            RandomVectorScorerSupplier supplier,
-            RandomAccessVectorValues vectorValues,
-            Closeable onClose
-        ) {
+        BinarizedCloseableRandomVectorScorerSupplier(RandomVectorScorerSupplier supplier, KnnVectorValues vectorValues, Closeable onClose) {
             this.supplier = supplier;
             this.onClose = onClose;
             this.vectorValues = vectorValues;
@@ -932,7 +963,6 @@ public int totalVectorCount() {
     static final class NormalizedFloatVectorValues extends FloatVectorValues {
         private final FloatVectorValues values;
         private final float[] normalizedVector;
-        int curDoc = -1;
 
         NormalizedFloatVectorValues(FloatVectorValues values) {
             this.values = values;
@@ -950,38 +980,25 @@ public int size() {
         }
 
         @Override
-        public float[] vectorValue() {
-            return normalizedVector;
+        public int ordToDoc(int ord) {
+            return values.ordToDoc(ord);
         }
 
         @Override
-        public VectorScorer scorer(float[] query) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public int docID() {
-            return values.docID();
+        public float[] vectorValue(int ord) throws IOException {
+            System.arraycopy(values.vectorValue(ord), 0, normalizedVector, 0, normalizedVector.length);
+            VectorUtil.l2normalize(normalizedVector);
+            return normalizedVector;
         }
 
         @Override
-        public int nextDoc() throws IOException {
-            curDoc = values.nextDoc();
-            if (curDoc != NO_MORE_DOCS) {
-                System.arraycopy(values.vectorValue(), 0, normalizedVector, 0, normalizedVector.length);
-                VectorUtil.l2normalize(normalizedVector);
-            }
-            return curDoc;
+        public DocIndexIterator iterator() {
+            return values.iterator();
         }
 
         @Override
-        public int advance(int target) throws IOException {
-            curDoc = values.advance(target);
-            if (curDoc != NO_MORE_DOCS) {
-                System.arraycopy(values.vectorValue(), 0, normalizedVector, 0, normalizedVector.length);
-                VectorUtil.l2normalize(normalizedVector);
-            }
-            return curDoc;
+        public NormalizedFloatVectorValues copy() throws IOException {
+            return new NormalizedFloatVectorValues(values.copy());
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java
index 2a3c3aca60e54..e7d818bb752d6 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java
@@ -34,9 +34,10 @@
 import java.nio.ByteBuffer;
 
 import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;
+import static org.elasticsearch.index.codec.vectors.BQVectorUtils.constSqrt;
 
 /** Binarized vector values loaded from off-heap */
-public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorValues implements RandomAccessBinarizedByteVectorValues {
+public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorValues {
 
     protected final int dimension;
     protected final int size;
@@ -53,6 +54,9 @@ public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorVa
     protected final BinaryQuantizer binaryQuantizer;
     protected final float[] centroid;
     protected final float centroidDp;
+    private final int discretizedDimensions;
+    private final float maxX1;
+    private final float sqrtDimensions;
     private final int correctionsCount;
 
     OffHeapBinarizedVectorValues(
@@ -79,6 +83,9 @@ public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorVa
         this.byteBuffer = ByteBuffer.allocate(numBytes);
         this.binaryValue = byteBuffer.array();
         this.binaryQuantizer = quantizer;
+        this.discretizedDimensions = BQVectorUtils.discretize(dimension, 64);
+        this.sqrtDimensions = (float) constSqrt(dimension);
+        this.maxX1 = (float) (1.9 / constSqrt(discretizedDimensions - 1.0));
     }
 
     @Override
@@ -103,13 +110,33 @@ public byte[] vectorValue(int targetOrd) throws IOException {
         return binaryValue;
     }
 
+    @Override
+    public int discretizedDimensions() {
+        return discretizedDimensions;
+    }
+
+    @Override
+    public float sqrtDimensions() {
+        return sqrtDimensions;
+    }
+
+    @Override
+    public float maxX1() {
+        return maxX1;
+    }
+
     @Override
     public float getCentroidDP() {
         return centroidDp;
     }
 
     @Override
-    public float[] getCorrectiveTerms() {
+    public float[] getCorrectiveTerms(int targetOrd) throws IOException {
+        if (lastOrd == targetOrd) {
+            return correctiveValues;
+        }
+        slice.seek(((long) targetOrd * byteSize) + numBytes);
+        slice.readFloats(correctiveValues, 0, correctionsCount);
         return correctiveValues;
     }
 
@@ -173,11 +200,6 @@ public float[] getCentroid() {
         return centroid;
     }
 
-    @Override
-    public IndexInput getSlice() {
-        return slice;
-    }
-
     @Override
     public int getVectorByteLength() {
         return numBytes;
@@ -230,8 +252,6 @@ public static OffHeapBinarizedVectorValues load(
 
     /** Dense off-heap binarized vector values */
     public static class DenseOffHeapVectorValues extends OffHeapBinarizedVectorValues {
-        private int doc = -1;
-
         public DenseOffHeapVectorValues(
             int dimension,
             int size,
@@ -245,30 +265,6 @@ public DenseOffHeapVectorValues(
             super(dimension, size, centroid, centroidDp, binaryQuantizer, similarityFunction, vectorsScorer, slice);
         }
 
-        @Override
-        public byte[] vectorValue() throws IOException {
-            return vectorValue(doc);
-        }
-
-        @Override
-        public int docID() {
-            return doc;
-        }
-
-        @Override
-        public int nextDoc() {
-            return advance(doc + 1);
-        }
-
-        @Override
-        public int advance(int target) {
-            assert docID() < target;
-            if (target >= size) {
-                return doc = NO_MORE_DOCS;
-            }
-            return doc = target;
-        }
-
         @Override
         public DenseOffHeapVectorValues copy() throws IOException {
             return new DenseOffHeapVectorValues(
@@ -291,19 +287,25 @@ public Bits getAcceptOrds(Bits acceptDocs) {
         @Override
         public VectorScorer scorer(float[] target) throws IOException {
             DenseOffHeapVectorValues copy = copy();
+            DocIndexIterator iterator = copy.iterator();
             RandomVectorScorer scorer = vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
             return new VectorScorer() {
                 @Override
                 public float score() throws IOException {
-                    return scorer.score(copy.doc);
+                    return scorer.score(iterator.index());
                 }
 
                 @Override
                 public DocIdSetIterator iterator() {
-                    return copy;
+                    return iterator;
                 }
             };
         }
+
+        @Override
+        public DocIndexIterator iterator() {
+            return createDenseIterator();
+        }
     }
 
     /** Sparse off-heap binarized vector values */
@@ -333,27 +335,6 @@ private static class SparseOffHeapVectorValues extends OffHeapBinarizedVectorVal
             this.disi = configuration.getIndexedDISI(dataIn);
         }
 
-        @Override
-        public byte[] vectorValue() throws IOException {
-            return vectorValue(disi.index());
-        }
-
-        @Override
-        public int docID() {
-            return disi.docID();
-        }
-
-        @Override
-        public int nextDoc() throws IOException {
-            return disi.nextDoc();
-        }
-
-        @Override
-        public int advance(int target) throws IOException {
-            assert docID() < target;
-            return disi.advance(target);
-        }
-
         @Override
         public SparseOffHeapVectorValues copy() throws IOException {
             return new SparseOffHeapVectorValues(
@@ -393,19 +374,25 @@ public int length() {
             };
         }
 
+        @Override
+        public DocIndexIterator iterator() {
+            return IndexedDISI.asDocIndexIterator(disi);
+        }
+
         @Override
         public VectorScorer scorer(float[] target) throws IOException {
             SparseOffHeapVectorValues copy = copy();
+            DocIndexIterator iterator = copy.iterator();
             RandomVectorScorer scorer = vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
             return new VectorScorer() {
                 @Override
                 public float score() throws IOException {
-                    return scorer.score(copy.disi.index());
+                    return scorer.score(iterator.index());
                 }
 
                 @Override
                 public DocIdSetIterator iterator() {
-                    return copy;
+                    return iterator;
                 }
             };
         }
@@ -419,23 +406,8 @@ private static class EmptyOffHeapVectorValues extends OffHeapBinarizedVectorValu
         }
 
         @Override
-        public int docID() {
-            return doc;
-        }
-
-        @Override
-        public int nextDoc() {
-            return advance(doc + 1);
-        }
-
-        @Override
-        public int advance(int target) {
-            return doc = NO_MORE_DOCS;
-        }
-
-        @Override
-        public byte[] vectorValue() {
-            throw new UnsupportedOperationException();
+        public DocIndexIterator iterator() {
+            return createDenseIterator();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java
deleted file mode 100644
index 2417353373ba5..0000000000000
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * @notice
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Modifications copyright (C) 2024 Elasticsearch B.V.
- */
-package org.elasticsearch.index.codec.vectors;
-
-import org.apache.lucene.util.VectorUtil;
-import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
-
-import java.io.IOException;
-
-/**
- * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10
- */
-public interface RandomAccessBinarizedByteVectorValues extends RandomAccessVectorValues.Bytes {
-    /** Returns the centroid distance for the vector */
-    float getCentroidDistance(int vectorOrd) throws IOException;
-
-    /** Returns the vector magnitude for the vector */
-    float getVectorMagnitude(int vectorOrd) throws IOException;
-
-    /** Returns OOQ corrective factor for the given vector ordinal */
-    float getOOQ(int targetOrd) throws IOException;
-
-    /**
-     * Returns the norm of the target vector w the centroid corrective factor for the given vector
-     * ordinal
-     */
-    float getNormOC(int targetOrd) throws IOException;
-
-    /**
-     * Returns the target vector dot product the centroid corrective factor for the given vector
-     * ordinal
-     */
-    float getODotC(int targetOrd) throws IOException;
-
-    /**
-     * @return the quantizer used to quantize the vectors
-     */
-    BinaryQuantizer getQuantizer();
-
-    /**
-     * @return coarse grained centroids for the vectors
-     */
-    float[] getCentroid() throws IOException;
-
-    @Override
-    RandomAccessBinarizedByteVectorValues copy() throws IOException;
-
-    default float getCentroidDP() throws IOException {
-        // this only gets executed on-merge
-        float[] centroid = getCentroid();
-        return VectorUtil.dotProduct(centroid, centroid);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
index d321600e03bf9..90f8e6adab73d 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
@@ -15,11 +15,7 @@
 import org.apache.lucene.index.MergeScheduler;
 import org.apache.lucene.util.SameThreadExecutorService;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexSettings;
@@ -29,8 +25,6 @@
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.Executor;
 
@@ -38,23 +32,13 @@
  * An extension to the {@link ConcurrentMergeScheduler} that provides tracking on merge times, total
  * and current merges.
  */
-class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler {
+public class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler implements ElasticsearchMergeScheduler {
 
     protected final Logger logger;
     private final Settings indexSettings;
     private final ShardId shardId;
 
-    private final MeanMetric totalMerges = new MeanMetric();
-    private final CounterMetric totalMergesNumDocs = new CounterMetric();
-    private final CounterMetric totalMergesSizeInBytes = new CounterMetric();
-    private final CounterMetric currentMerges = new CounterMetric();
-    private final CounterMetric currentMergesNumDocs = new CounterMetric();
-    private final CounterMetric currentMergesSizeInBytes = new CounterMetric();
-    private final CounterMetric totalMergeStoppedTime = new CounterMetric();
-    private final CounterMetric totalMergeThrottledTime = new CounterMetric();
-
-    private final Set<OnGoingMerge> onGoingMerges = ConcurrentCollections.newConcurrentSet();
-    private final Set<OnGoingMerge> readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges);
+    private final MergeTracking mergeTracking;
     private final MergeSchedulerConfig config;
     private final SameThreadExecutorService sameThreadExecutorService = new SameThreadExecutorService();
 
@@ -63,11 +47,16 @@ class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler {
         this.shardId = shardId;
         this.indexSettings = indexSettings.getSettings();
         this.logger = Loggers.getLogger(getClass(), shardId);
+        this.mergeTracking = new MergeTracking(
+            logger,
+            () -> indexSettings.getMergeSchedulerConfig().isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY
+        );
         refreshConfig();
     }
 
+    @Override
     public Set<OnGoingMerge> onGoingMerges() {
-        return readOnlyOnGoingMerges;
+        return mergeTracking.onGoingMerges();
     }
 
     /** We're currently only interested in messages with this prefix. */
@@ -104,74 +93,21 @@ protected void message(String message) {
         super.message(message);
     }
 
-    private static String getSegmentName(MergePolicy.OneMerge merge) {
-        return merge.getMergeInfo() != null ? merge.getMergeInfo().info.name : "_na_";
-    }
-
     @Override
     protected void doMerge(MergeSource mergeSource, MergePolicy.OneMerge merge) throws IOException {
-        int totalNumDocs = merge.totalNumDocs();
-        long totalSizeInBytes = merge.totalBytesSize();
         long timeNS = System.nanoTime();
-        currentMerges.inc();
-        currentMergesNumDocs.inc(totalNumDocs);
-        currentMergesSizeInBytes.inc(totalSizeInBytes);
-
         OnGoingMerge onGoingMerge = new OnGoingMerge(merge);
-        onGoingMerges.add(onGoingMerge);
-
-        if (logger.isTraceEnabled()) {
-            logger.trace(
-                "merge [{}] starting..., merging [{}] segments, [{}] docs, [{}] size, into [{}] estimated_size",
-                getSegmentName(merge),
-                merge.segments.size(),
-                totalNumDocs,
-                ByteSizeValue.ofBytes(totalSizeInBytes),
-                ByteSizeValue.ofBytes(merge.estimatedMergeBytes)
-            );
-        }
+        mergeTracking.mergeStarted(onGoingMerge);
         try {
             beforeMerge(onGoingMerge);
             super.doMerge(mergeSource, merge);
         } finally {
             long tookMS = TimeValue.nsecToMSec(System.nanoTime() - timeNS);
+            mergeTracking.mergeFinished(merge, onGoingMerge, tookMS);
 
-            onGoingMerges.remove(onGoingMerge);
             afterMerge(onGoingMerge);
-
-            currentMerges.dec();
-            currentMergesNumDocs.dec(totalNumDocs);
-            currentMergesSizeInBytes.dec(totalSizeInBytes);
-
-            totalMergesNumDocs.inc(totalNumDocs);
-            totalMergesSizeInBytes.inc(totalSizeInBytes);
-            totalMerges.inc(tookMS);
-            long stoppedMS = TimeValue.nsecToMSec(
-                merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.STOPPED)
-            );
-            long throttledMS = TimeValue.nsecToMSec(
-                merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.PAUSED)
-            );
-            totalMergeStoppedTime.inc(stoppedMS);
-            totalMergeThrottledTime.inc(throttledMS);
-
-            String message = String.format(
-                Locale.ROOT,
-                "merge segment [%s] done: took [%s], [%,.1f MB], [%,d docs], [%s stopped], [%s throttled]",
-                getSegmentName(merge),
-                TimeValue.timeValueMillis(tookMS),
-                totalSizeInBytes / 1024f / 1024f,
-                totalNumDocs,
-                TimeValue.timeValueMillis(stoppedMS),
-                TimeValue.timeValueMillis(throttledMS)
-            );
-
-            if (tookMS > 20000) { // if more than 20 seconds, DEBUG log it
-                logger.debug("{}", message);
-            } else if (logger.isTraceEnabled()) {
-                logger.trace("{}", message);
-            }
         }
+
     }
 
     /**
@@ -206,24 +142,13 @@ protected MergeThread getMergeThread(MergeSource mergeSource, MergePolicy.OneMer
         return thread;
     }
 
-    MergeStats stats() {
-        final MergeStats mergeStats = new MergeStats();
-        mergeStats.add(
-            totalMerges.count(),
-            totalMerges.sum(),
-            totalMergesNumDocs.count(),
-            totalMergesSizeInBytes.count(),
-            currentMerges.count(),
-            currentMergesNumDocs.count(),
-            currentMergesSizeInBytes.count(),
-            totalMergeStoppedTime.count(),
-            totalMergeThrottledTime.count(),
-            config.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY
-        );
-        return mergeStats;
+    @Override
+    public MergeStats stats() {
+        return mergeTracking.stats();
     }
 
-    void refreshConfig() {
+    @Override
+    public void refreshConfig() {
         if (this.getMaxMergeCount() != config.getMaxMergeCount() || this.getMaxThreadCount() != config.getMaxThreadCount()) {
             this.setMaxMergesAndThreads(config.getMaxMergeCount(), config.getMaxThreadCount());
         }
@@ -234,4 +159,9 @@ void refreshConfig() {
             disableAutoIOThrottle();
         }
     }
+
+    @Override
+    public MergeScheduler getMergeScheduler() {
+        return this;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchMergeScheduler.java
similarity index 52%
rename from server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java
rename to server/src/main/java/org/elasticsearch/index/engine/ElasticsearchMergeScheduler.java
index 69ea4484ae691..ac72c7a21da75 100644
--- a/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchMergeScheduler.java
@@ -7,21 +7,21 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.action.search;
+package org.elasticsearch.index.engine;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.apache.lucene.index.MergeScheduler;
+import org.elasticsearch.index.merge.MergeStats;
+import org.elasticsearch.index.merge.OnGoingMerge;
 
-import java.io.IOException;
+import java.util.Set;
 
-public class VersionMismatchException extends ElasticsearchException {
+public interface ElasticsearchMergeScheduler {
 
-    public VersionMismatchException(String msg, Object... args) {
-        super(msg, args);
-    }
+    Set<OnGoingMerge> onGoingMerges();
 
-    public VersionMismatchException(StreamInput in) throws IOException {
-        super(in);
-    }
+    MergeStats stats();
 
+    void refreshConfig();
+
+    MergeScheduler getMergeScheduler();
 }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index c72f5ce740d94..8d43252d178ee 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -20,6 +20,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.LiveIndexWriterConfig;
 import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.MergeScheduler;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
@@ -139,7 +140,7 @@ public class InternalEngine extends Engine {
     private volatile long lastDeleteVersionPruneTimeMSec;
 
     private final Translog translog;
-    private final ElasticsearchConcurrentMergeScheduler mergeScheduler;
+    private final ElasticsearchMergeScheduler mergeScheduler;
 
     private final IndexWriter indexWriter;
 
@@ -248,11 +249,12 @@ public InternalEngine(EngineConfig engineConfig) {
         Translog translog = null;
         ExternalReaderManager externalReaderManager = null;
         ElasticsearchReaderManager internalReaderManager = null;
-        EngineMergeScheduler scheduler = null;
+        MergeScheduler scheduler = null;
         boolean success = false;
         try {
             this.lastDeleteVersionPruneTimeMSec = engineConfig.getThreadPool().relativeTimeInMillis();
-            mergeScheduler = scheduler = new EngineMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings());
+            mergeScheduler = createMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings());
+            scheduler = mergeScheduler.getMergeScheduler();
             throttle = new IndexThrottle();
             try {
                 store.trimUnsafeCommits(config().getTranslogConfig().getTranslogPath());
@@ -383,7 +385,7 @@ private SoftDeletesPolicy newSoftDeletesPolicy() throws IOException {
 
     @Nullable
     private CombinedDeletionPolicy.CommitsListener newCommitsListener() {
-        Engine.IndexCommitListener listener = engineConfig.getIndexCommitListener();
+        IndexCommitListener listener = engineConfig.getIndexCommitListener();
         if (listener != null) {
             final IndexCommitListener wrappedListener = Assertions.ENABLED ? assertingCommitsOrderListener(listener) : listener;
             return new CombinedDeletionPolicy.CommitsListener() {
@@ -824,7 +826,7 @@ private GetResult getFromTranslog(
             config(),
             translogInMemorySegmentsCount::incrementAndGet
         );
-        final Engine.Searcher searcher = new Engine.Searcher(
+        final Searcher searcher = new Searcher(
             "realtime_get",
             ElasticsearchDirectoryReader.wrap(inMemoryReader, shardId),
             config().getSimilarity(),
@@ -841,7 +843,7 @@ public GetResult get(
         Get get,
         MappingLookup mappingLookup,
         DocumentParser documentParser,
-        Function<Engine.Searcher, Engine.Searcher> searcherWrapper
+        Function<Searcher, Searcher> searcherWrapper
     ) {
         try (var ignored = acquireEnsureOpenRef()) {
             if (get.realtime()) {
@@ -875,7 +877,7 @@ protected GetResult realtimeGetUnderLock(
         Get get,
         MappingLookup mappingLookup,
         DocumentParser documentParser,
-        Function<Engine.Searcher, Engine.Searcher> searcherWrapper,
+        Function<Searcher, Searcher> searcherWrapper,
         boolean getFromSearcher
     ) {
         assert isDrainedForClose() == false;
@@ -1098,7 +1100,7 @@ protected boolean assertPrimaryCanOptimizeAddDocument(final Index index) {
         return true;
     }
 
-    private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) {
+    private boolean assertIncomingSequenceNumber(final Operation.Origin origin, final long seqNo) {
         if (origin == Operation.Origin.PRIMARY) {
             assert assertPrimaryIncomingSequenceNumber(origin, seqNo);
         } else {
@@ -1108,7 +1110,7 @@ private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origi
         return true;
     }
 
-    protected boolean assertPrimaryIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) {
+    protected boolean assertPrimaryIncomingSequenceNumber(final Operation.Origin origin, final long seqNo) {
         // sequence number should not be set when operation origin is primary
         assert seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO
             : "primary operations must never have an assigned sequence number but was [" + seqNo + "]";
@@ -2700,7 +2702,7 @@ private IndexWriterConfig getIndexWriterConfig() {
         iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
         iwc.setIndexDeletionPolicy(combinedDeletionPolicy);
         iwc.setInfoStream(TESTS_VERBOSE ? InfoStream.getDefault() : new LoggerInfoStream(logger));
-        iwc.setMergeScheduler(mergeScheduler);
+        iwc.setMergeScheduler(mergeScheduler.getMergeScheduler());
         // Give us the opportunity to upgrade old segments while performing
         // background merges
         MergePolicy mergePolicy = config().getMergePolicy();
@@ -2753,7 +2755,7 @@ private IndexWriterConfig getIndexWriterConfig() {
 
     /** A listener that warms the segments if needed when acquiring a new reader */
     static final class RefreshWarmerListener implements BiConsumer<ElasticsearchDirectoryReader, ElasticsearchDirectoryReader> {
-        private final Engine.Warmer warmer;
+        private final Warmer warmer;
         private final Logger logger;
         private final AtomicBoolean isEngineClosed;
 
@@ -2817,6 +2819,10 @@ LiveIndexWriterConfig getCurrentIndexWriterConfig() {
         return indexWriter.getConfig();
     }
 
+    protected ElasticsearchMergeScheduler createMergeScheduler(ShardId shardId, IndexSettings indexSettings) {
+        return new EngineMergeScheduler(shardId, indexSettings);
+    }
+
     private final class EngineMergeScheduler extends ElasticsearchConcurrentMergeScheduler {
         private final AtomicInteger numMergesInFlight = new AtomicInteger(0);
         private final AtomicBoolean isThrottling = new AtomicBoolean();
@@ -2827,7 +2833,7 @@ private final class EngineMergeScheduler extends ElasticsearchConcurrentMergeSch
 
         @Override
         public synchronized void beforeMerge(OnGoingMerge merge) {
-            int maxNumMerges = mergeScheduler.getMaxMergeCount();
+            int maxNumMerges = getMaxMergeCount();
             if (numMergesInFlight.incrementAndGet() > maxNumMerges) {
                 if (isThrottling.getAndSet(true) == false) {
                     logger.info("now throttling indexing: numMergesInFlight={}, maxNumMerges={}", numMergesInFlight, maxNumMerges);
@@ -2838,7 +2844,7 @@ public synchronized void beforeMerge(OnGoingMerge merge) {
 
         @Override
         public synchronized void afterMerge(OnGoingMerge merge) {
-            int maxNumMerges = mergeScheduler.getMaxMergeCount();
+            int maxNumMerges = getMaxMergeCount();
             if (numMergesInFlight.decrementAndGet() < maxNumMerges) {
                 if (isThrottling.getAndSet(false)) {
                     logger.info("stop throttling indexing: numMergesInFlight={}, maxNumMerges={}", numMergesInFlight, maxNumMerges);
@@ -2876,25 +2882,29 @@ protected void doRun() {
 
         @Override
         protected void handleMergeException(final Throwable exc) {
-            engineConfig.getThreadPool().generic().execute(new AbstractRunnable() {
-                @Override
-                public void onFailure(Exception e) {
-                    logger.debug("merge failure action rejected", e);
-                }
-
-                @Override
-                protected void doRun() throws Exception {
-                    /*
-                     * We do this on another thread rather than the merge thread that we are initially called on so that we have complete
-                     * confidence that the call stack does not contain catch statements that would cause the error that might be thrown
-                     * here from being caught and never reaching the uncaught exception handler.
-                     */
-                    failEngine("merge failed", new MergePolicy.MergeException(exc));
-                }
-            });
+            mergeException(exc);
         }
     }
 
+    protected void mergeException(final Throwable exc) {
+        engineConfig.getThreadPool().generic().execute(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                logger.debug("merge failure action rejected", e);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                /*
+                 * We do this on another thread rather than the merge thread that we are initially called on so that we have complete
+                 * confidence that the call stack does not contain catch statements that would cause the error that might be thrown
+                 * here from being caught and never reaching the uncaught exception handler.
+                 */
+                failEngine("merge failed", new MergePolicy.MergeException(exc));
+            }
+        });
+    }
+
     /**
      * Commits the specified index writer.
      *
diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
index 05cc6d148be5e..e44b344d3b283 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
@@ -119,7 +119,7 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
         this.parallelArray = new ParallelArray(this.searchBatchSize);
         this.indexVersionCreated = indexVersionCreated;
         final TopDocs topDocs = searchOperations(null, accessStats);
-        this.totalHits = Math.toIntExact(topDocs.totalHits.value);
+        this.totalHits = Math.toIntExact(topDocs.totalHits.value());
         this.scoreDocs = topDocs.scoreDocs;
         fillParallelArray(scoreDocs, parallelArray);
     }
@@ -341,7 +341,7 @@ private Translog.Operation readDocAsOp(int docIndex) throws IOException {
             assert storedFieldsReaderOrd == leaf.ord : storedFieldsReaderOrd + " != " + leaf.ord;
             storedFieldsReader.document(segmentDocID, fields);
         } else {
-            leaf.reader().document(segmentDocID, fields);
+            leaf.reader().storedFields().document(segmentDocID, fields);
         }
 
         final Translog.Operation op;
diff --git a/server/src/main/java/org/elasticsearch/index/engine/MergeTracking.java b/server/src/main/java/org/elasticsearch/index/engine/MergeTracking.java
new file mode 100644
index 0000000000000..3f52b607cf356
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/engine/MergeTracking.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.MergePolicy;
+import org.elasticsearch.common.metrics.CounterMetric;
+import org.elasticsearch.common.metrics.MeanMetric;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.merge.MergeStats;
+import org.elasticsearch.index.merge.OnGoingMerge;
+
+import java.util.Collections;
+import java.util.Locale;
+import java.util.Set;
+import java.util.function.DoubleSupplier;
+
+public class MergeTracking {
+
+    protected final Logger logger;
+    private final DoubleSupplier mbPerSecAutoThrottle;
+
+    private final MeanMetric totalMerges = new MeanMetric();
+    private final CounterMetric totalMergesNumDocs = new CounterMetric();
+    private final CounterMetric totalMergesSizeInBytes = new CounterMetric();
+    private final CounterMetric currentMerges = new CounterMetric();
+    private final CounterMetric currentMergesNumDocs = new CounterMetric();
+    private final CounterMetric currentMergesSizeInBytes = new CounterMetric();
+    private final CounterMetric totalMergeStoppedTime = new CounterMetric();
+    private final CounterMetric totalMergeThrottledTime = new CounterMetric();
+
+    private final Set<OnGoingMerge> onGoingMerges = ConcurrentCollections.newConcurrentSet();
+    private final Set<OnGoingMerge> readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges);
+
+    public MergeTracking(Logger logger, DoubleSupplier mbPerSecAutoThrottle) {
+        this.logger = logger;
+        this.mbPerSecAutoThrottle = mbPerSecAutoThrottle;
+    }
+
+    public Set<OnGoingMerge> onGoingMerges() {
+        return readOnlyOnGoingMerges;
+    }
+
+    public void mergeStarted(OnGoingMerge onGoingMerge) {
+        MergePolicy.OneMerge merge = onGoingMerge.getMerge();
+        int totalNumDocs = merge.totalNumDocs();
+        long totalSizeInBytes = merge.totalBytesSize();
+        currentMerges.inc();
+        currentMergesNumDocs.inc(totalNumDocs);
+        currentMergesSizeInBytes.inc(totalSizeInBytes);
+        onGoingMerges.add(onGoingMerge);
+
+        if (logger.isTraceEnabled()) {
+            logger.trace(
+                "merge [{}] starting: merging [{}] segments, [{}] docs, [{}] size, into [{}] estimated_size",
+                onGoingMerge.getId(),
+                merge.segments.size(),
+                totalNumDocs,
+                ByteSizeValue.ofBytes(totalSizeInBytes),
+                ByteSizeValue.ofBytes(merge.estimatedMergeBytes)
+            );
+        }
+    }
+
+    public void mergeFinished(final MergePolicy.OneMerge merge, final OnGoingMerge onGoingMerge, long tookMS) {
+        int totalNumDocs = merge.totalNumDocs();
+        long totalSizeInBytes = merge.totalBytesSize();
+
+        onGoingMerges.remove(onGoingMerge);
+
+        currentMerges.dec();
+        currentMergesNumDocs.dec(totalNumDocs);
+        currentMergesSizeInBytes.dec(totalSizeInBytes);
+
+        totalMergesNumDocs.inc(totalNumDocs);
+        totalMergesSizeInBytes.inc(totalSizeInBytes);
+        totalMerges.inc(tookMS);
+        long stoppedMS = TimeValue.nsecToMSec(
+            merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.STOPPED)
+        );
+        long throttledMS = TimeValue.nsecToMSec(
+            merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.PAUSED)
+        );
+        totalMergeStoppedTime.inc(stoppedMS);
+        totalMergeThrottledTime.inc(throttledMS);
+
+        String message = String.format(
+            Locale.ROOT,
+            "merge [%s] segment [%s] done: took [%s], [%s], [%,d] docs, [%s] stopped, [%s] throttled",
+            onGoingMerge.getId(),
+            getSegmentName(merge),
+            TimeValue.timeValueMillis(tookMS),
+            ByteSizeValue.ofBytes(totalSizeInBytes),
+            totalNumDocs,
+            TimeValue.timeValueMillis(stoppedMS),
+            TimeValue.timeValueMillis(throttledMS)
+        );
+
+        if (tookMS > 20000) { // if more than 20 seconds, DEBUG log it
+            logger.debug("{}", message);
+        } else if (logger.isTraceEnabled()) {
+            logger.trace("{}", message);
+        }
+    }
+
+    public MergeStats stats() {
+        final MergeStats mergeStats = new MergeStats();
+        mergeStats.add(
+            totalMerges.count(),
+            totalMerges.sum(),
+            totalMergesNumDocs.count(),
+            totalMergesSizeInBytes.count(),
+            currentMerges.count(),
+            currentMergesNumDocs.count(),
+            currentMergesSizeInBytes.count(),
+            totalMergeStoppedTime.count(),
+            totalMergeThrottledTime.count(),
+            mbPerSecAutoThrottle.getAsDouble()
+        );
+        return mergeStats;
+    }
+
+    private static String getSegmentName(MergePolicy.OneMerge merge) {
+        return merge.getMergeInfo() != null ? merge.getMergeInfo().info.name : "_na_";
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java
index 18b5ba69ca320..3e99818d1827b 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FilterCodecReader;
 import org.apache.lucene.index.FilterNumericDocValues;
@@ -188,6 +189,11 @@ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
                 return in.getSortedSet(field);
             }
 
+            @Override
+            public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
+                return in.getSkipper(field);
+            }
+
             @Override
             public void checkIntegrity() throws IOException {
                 in.checkIntegrity();
diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
index c7acd730fadb5..0f772b49bf92b 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
@@ -13,10 +13,11 @@
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesSkipIndexType;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.IndexCommit;
@@ -152,6 +153,7 @@ private static class TranslogLeafReader extends LeafReader {
             false,
             IndexOptions.NONE,
             DocValuesType.NONE,
+            DocValuesSkipIndexType.NONE,
             -1,
             Collections.emptyMap(),
             0,
@@ -171,6 +173,7 @@ private static class TranslogLeafReader extends LeafReader {
             false,
             IndexOptions.NONE,
             DocValuesType.NONE,
+            DocValuesSkipIndexType.NONE,
             -1,
             Collections.emptyMap(),
             0,
@@ -190,6 +193,7 @@ private static class TranslogLeafReader extends LeafReader {
             false,
             IndexOptions.DOCS,
             DocValuesType.NONE,
+            DocValuesSkipIndexType.NONE,
             -1,
             Collections.emptyMap(),
             0,
@@ -346,6 +350,11 @@ public NumericDocValues getNormValues(String field) throws IOException {
             return getDelegate().getNormValues(field);
         }
 
+        @Override
+        public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {
+            return getDelegate().getDocValuesSkipper(field);
+        }
+
         @Override
         public FloatVectorValues getFloatVectorValues(String field) throws IOException {
             return getDelegate().getFloatVectorValues(field);
@@ -389,11 +398,6 @@ public LeafMetaData getMetaData() {
             return getDelegate().getMetaData();
         }
 
-        @Override
-        public Fields getTermVectors(int docID) throws IOException {
-            return getDelegate().getTermVectors(docID);
-        }
-
         @Override
         public TermVectors termVectors() throws IOException {
             return getDelegate().termVectors();
@@ -429,11 +433,6 @@ public int maxDoc() {
             return 1;
         }
 
-        @Override
-        public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-            storedFields().document(docID, visitor);
-        }
-
         private void readStoredFieldsDirectly(StoredFieldVisitor visitor) throws IOException {
             if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) {
                 BytesReference sourceBytes = operation.source();
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
index 84e85f3ddf2b4..d4e34181b876f 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
@@ -52,12 +52,7 @@ public boolean advanceExact(int target) throws IOException {
 
     @Override
     public long nextOrd() throws IOException {
-        long segmentOrd = values.nextOrd();
-        if (segmentOrd == SortedSetDocValues.NO_MORE_ORDS) {
-            return SortedSetDocValues.NO_MORE_ORDS;
-        } else {
-            return getGlobalOrd(segmentOrd);
-        }
+        return getGlobalOrd(values.nextOrd());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java
index 0439383ccbd05..0f72e491d8110 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java
@@ -40,13 +40,13 @@ public static boolean significantlySmallerThanSinglePackedOrdinals(
         float acceptableOverheadRatio
     ) {
         int bitsPerOrd = PackedInts.bitsRequired(numOrds);
-        bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue;
+        bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue();
         // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the
         // beginning of the block and all docs have one at the end of the block
         final float avgValuesPerDoc = (float) numDocsWithValue / maxDoc;
         final int maxDelta = (int) Math.ceil(OFFSETS_PAGE_SIZE * (1 - avgValuesPerDoc) * avgValuesPerDoc);
         int bitsPerOffset = PackedInts.bitsRequired(maxDelta) + 1; // +1 because of the sign
-        bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue;
+        bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue();
 
         final long expectedMultiSizeInBytes = (long) numDocsWithValue * bitsPerOrd + (long) maxDoc * bitsPerOffset;
         final long expectedSingleSizeInBytes = (long) maxDoc * bitsPerOrd;
@@ -153,6 +153,7 @@ private static class MultiDocs extends AbstractSortedSetDocValues {
 
         private long currentOffset;
         private long currentEndOffset;
+        private int count;
 
         MultiDocs(MultiOrdinals ordinals, ValuesHolder values) {
             this.valueCount = ordinals.valueCount;
@@ -170,21 +171,19 @@ public long getValueCount() {
         public boolean advanceExact(int docId) {
             currentOffset = docId != 0 ? endOffsets.get(docId - 1) : 0;
             currentEndOffset = endOffsets.get(docId);
+            count = Math.toIntExact(currentEndOffset - currentOffset);
             return currentOffset != currentEndOffset;
         }
 
         @Override
         public long nextOrd() {
-            if (currentOffset == currentEndOffset) {
-                return SortedSetDocValues.NO_MORE_ORDS;
-            } else {
-                return ords.get(currentOffset++);
-            }
+            assert currentOffset != currentEndOffset;
+            return ords.get(currentOffset++);
         }
 
         @Override
         public int docValueCount() {
-            return Math.toIntExact(currentEndOffset - currentOffset);
+            return count;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index 402f455d69bc2..109f645f24caf 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -20,11 +20,9 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.rest.action.document.RestMultiGetAction;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -276,9 +274,6 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(_INDEX, index);
-        if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-            builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-        }
         builder.field(_ID, id);
         if (isExists()) {
             if (version != -1) {
@@ -316,8 +311,6 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
             } else if (token.isValue()) {
                 if (_INDEX.equals(currentFieldName)) {
                     index = parser.text();
-                } else if (parser.getRestApiVersion() == RestApiVersion.V_7 && MapperService.TYPE_FIELD_NAME.equals(currentFieldName)) {
-                    deprecationLogger.compatibleCritical("mget_with_types", RestMultiGetAction.TYPES_DEPRECATION_MESSAGE);
                 } else if (_ID.equals(currentFieldName)) {
                     id = parser.text();
                 } else if (_VERSION.equals(currentFieldName)) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
index 06bf66a4a09c6..87c123d71aae5 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
@@ -49,13 +49,13 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
-        private final boolean isSyntheticSourceEnabledViaIndexMode;
+        private final boolean isSyntheticSourceEnabled;
         private final Parameter<Boolean> hasDocValues;
 
-        public Builder(String name, boolean isSyntheticSourceEnabledViaIndexMode) {
+        public Builder(String name, boolean isSyntheticSourceEnabled) {
             super(name);
-            this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode;
-            this.hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, isSyntheticSourceEnabledViaIndexMode);
+            this.isSyntheticSourceEnabled = isSyntheticSourceEnabled;
+            this.hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, isSyntheticSourceEnabled);
         }
 
         @Override
@@ -79,9 +79,7 @@ public BinaryFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.getIndexSettings().getMode().isSyntheticSourceEnabled())
-    );
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, SourceFieldMapper.isSynthetic(c.getIndexSettings())));
 
     public static final class BinaryFieldType extends MappedFieldType {
         private BinaryFieldType(String name, boolean isStored, boolean hasDocValues, Map<String, String> meta) {
@@ -140,13 +138,13 @@ public Query termQuery(Object value, SearchExecutionContext context) {
 
     private final boolean stored;
     private final boolean hasDocValues;
-    private final boolean isSyntheticSourceEnabledViaIndexMode;
+    private final boolean isSyntheticSourceEnabled;
 
     protected BinaryFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, Builder builder) {
         super(simpleName, mappedFieldType, builderParams);
         this.stored = builder.stored.getValue();
         this.hasDocValues = builder.hasDocValues.getValue();
-        this.isSyntheticSourceEnabledViaIndexMode = builder.isSyntheticSourceEnabledViaIndexMode;
+        this.isSyntheticSourceEnabled = builder.isSyntheticSourceEnabled;
     }
 
     @Override
@@ -186,7 +184,7 @@ public void indexValue(DocumentParserContext context, byte[] value) {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new BinaryFieldMapper.Builder(leafName(), isSyntheticSourceEnabledViaIndexMode).init(this);
+        return new BinaryFieldMapper.Builder(leafName(), isSyntheticSourceEnabled).init(this);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
index 7be5ee2200b5c..87e4ce5f90479 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -11,6 +11,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -687,7 +688,7 @@ public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionCo
             long pivotLong = resolution.convert(pivotTime);
             // As we already apply boost in AbstractQueryBuilder::toQuery, we always passing a boost of 1.0 to distanceFeatureQuery
             if (isIndexed()) {
-                return LongPoint.newDistanceFeatureQuery(name(), 1.0f, originLong, pivotLong);
+                return LongField.newDistanceFeatureQuery(name(), 1.0f, originLong, pivotLong);
             } else {
                 return new LongScriptFieldDistanceFeatureQuery(
                     new Script(""),
@@ -958,7 +959,7 @@ private void indexValue(DocumentParserContext context, long timestamp) {
         }
 
         if (indexed && hasDocValues) {
-            context.doc().add(new LongField(fieldType().name(), timestamp));
+            context.doc().add(new LongField(fieldType().name(), timestamp, Field.Store.NO));
         } else if (hasDocValues) {
             context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp));
         } else if (indexed) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java
index 494005ce12cb1..d37f6c51d288d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java
@@ -11,10 +11,11 @@
 
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.DocValuesSkipIndexType;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
@@ -147,11 +148,6 @@ public FieldInfos getFieldInfos() {
         return new FieldInfos(new FieldInfo[0]);
     }
 
-    @Override
-    public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-        storedFields().document(docID, visitor);
-    }
-
     @Override
     public StoredFields storedFields() throws IOException {
         return new StoredFields() {
@@ -203,6 +199,11 @@ public NumericDocValues getNormValues(String field) throws IOException {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public DocValuesSkipper getDocValuesSkipper(String s) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public FloatVectorValues getFloatVectorValues(String field) throws IOException {
         throw new UnsupportedOperationException();
@@ -233,11 +234,6 @@ public LeafMetaData getMetaData() {
         throw new UnsupportedOperationException();
     }
 
-    @Override
-    public Fields getTermVectors(int docID) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public int numDocs() {
         throw new UnsupportedOperationException();
@@ -284,6 +280,7 @@ private static FieldInfo fieldInfo(String name) {
             false,
             IndexOptions.NONE,
             DocValuesType.NONE,
+            DocValuesSkipIndexType.NONE,
             -1,
             Collections.emptyMap(),
             0,
@@ -484,9 +481,7 @@ private static SortedSetDocValues sortedSetDocValues(List<BytesRef> values) {
             @Override
             public long nextOrd() {
                 i++;
-                if (i >= values.size()) {
-                    return NO_MORE_ORDS;
-                }
+                assert i < values.size();
                 return i;
             }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 0ff754d953934..1ed0a117ddd89 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -208,6 +208,7 @@ private static List<IgnoredSourceFieldMapper.NameValue> parseDocForMissingValues
         XContentParser parser = context.parser();
         XContentParser.Token currentToken = parser.nextToken();
         List<String> path = new ArrayList<>();
+        List<Boolean> isObjectInPath = new ArrayList<>();  // Tracks if path components correspond to an object or an array.
         String fieldName = null;
         while (currentToken != null) {
             while (currentToken != XContentParser.Token.FIELD_NAME) {
@@ -218,11 +219,16 @@ private static List<IgnoredSourceFieldMapper.NameValue> parseDocForMissingValues
                         parser.skipChildren();
                     } else {
                         path.add(fieldName);
+                        isObjectInPath.add(currentToken == XContentParser.Token.START_OBJECT);
                     }
                     fieldName = null;
                 } else if (currentToken == XContentParser.Token.END_OBJECT || currentToken == XContentParser.Token.END_ARRAY) {
-                    if (currentToken == XContentParser.Token.END_OBJECT && path.isEmpty() == false) {
+                    // Remove the path, if the scope type matches the one when the path was added.
+                    if (isObjectInPath.isEmpty() == false
+                        && (isObjectInPath.getLast() && currentToken == XContentParser.Token.END_OBJECT
+                            || isObjectInPath.getLast() == false && currentToken == XContentParser.Token.END_ARRAY)) {
                         path.removeLast();
+                        isObjectInPath.removeLast();
                     }
                     fieldName = null;
                 }
@@ -237,7 +243,6 @@ private static List<IgnoredSourceFieldMapper.NameValue> parseDocForMissingValues
             if (leaf != null) {
                 parser.nextToken();  // Advance the parser to the value to be read.
                 result.add(leaf.cloneWithValue(context.encodeFlattenedToken()));
-                parser.nextToken();  // Skip the token ending the value.
                 fieldName = null;
             }
             currentToken = parser.nextToken();
@@ -805,8 +810,10 @@ private static void parseNonDynamicArray(
             boolean objectWithFallbackSyntheticSource = false;
             if (mapper instanceof ObjectMapper objectMapper) {
                 mode = getSourceKeepMode(context, objectMapper.sourceKeepMode());
-                objectWithFallbackSyntheticSource = (mode == Mapper.SourceKeepMode.ALL
-                    || (mode == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false));
+                objectWithFallbackSyntheticSource = mode == Mapper.SourceKeepMode.ALL
+                    // Inside nested objects we always store object arrays as a workaround for #115261.
+                    || ((context.inNestedScope() || mode == Mapper.SourceKeepMode.ARRAYS)
+                        && objectMapper instanceof NestedObjectMapper == false);
             }
             boolean fieldWithFallbackSyntheticSource = false;
             boolean fieldWithStoredArraySource = false;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
index ac236e5a7e5fd..3b1f1a6d2809a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
@@ -104,6 +104,16 @@ public int get() {
         }
     }
 
+    /**
+     * Defines the scope parser is currently in.
+     * This is used for synthetic source related logic during parsing.
+     */
+    private enum Scope {
+        SINGLETON,
+        ARRAY,
+        NESTED
+    }
+
     private final MappingLookup mappingLookup;
     private final MappingParserContext mappingParserContext;
     private final SourceToParse sourceToParse;
@@ -111,7 +121,8 @@ public int get() {
     private final Set<String> ignoredFields;
     private final List<IgnoredSourceFieldMapper.NameValue> ignoredFieldValues;
     private final List<IgnoredSourceFieldMapper.NameValue> ignoredFieldsMissingValues;
-    private boolean inArrayScope;
+    private boolean inArrayScopeEnabled;
+    private Scope currentScope;
 
     private final Map<String, List<Mapper>> dynamicMappers;
     private final DynamicMapperSize dynamicMappersSize;
@@ -143,7 +154,8 @@ private DocumentParserContext(
         Set<String> ignoreFields,
         List<IgnoredSourceFieldMapper.NameValue> ignoredFieldValues,
         List<IgnoredSourceFieldMapper.NameValue> ignoredFieldsWithNoSource,
-        boolean inArrayScope,
+        boolean inArrayScopeEnabled,
+        Scope currentScope,
         Map<String, List<Mapper>> dynamicMappers,
         Map<String, ObjectMapper> dynamicObjectMappers,
         Map<String, List<RuntimeField>> dynamicRuntimeFields,
@@ -164,7 +176,8 @@ private DocumentParserContext(
         this.ignoredFields = ignoreFields;
         this.ignoredFieldValues = ignoredFieldValues;
         this.ignoredFieldsMissingValues = ignoredFieldsWithNoSource;
-        this.inArrayScope = inArrayScope;
+        this.inArrayScopeEnabled = inArrayScopeEnabled;
+        this.currentScope = currentScope;
         this.dynamicMappers = dynamicMappers;
         this.dynamicObjectMappers = dynamicObjectMappers;
         this.dynamicRuntimeFields = dynamicRuntimeFields;
@@ -188,7 +201,8 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic,
             in.ignoredFields,
             in.ignoredFieldValues,
             in.ignoredFieldsMissingValues,
-            in.inArrayScope,
+            in.inArrayScopeEnabled,
+            in.currentScope,
             in.dynamicMappers,
             in.dynamicObjectMappers,
             in.dynamicRuntimeFields,
@@ -219,7 +233,8 @@ protected DocumentParserContext(
             new HashSet<>(),
             new ArrayList<>(),
             new ArrayList<>(),
-            false,
+            mappingParserContext.getIndexSettings().isSyntheticSourceSecondDocParsingPassEnabled(),
+            Scope.SINGLETON,
             new HashMap<>(),
             new HashMap<>(),
             new HashMap<>(),
@@ -330,7 +345,7 @@ public final void deduplicateIgnoredFieldValues(final Set<String> fullNames) {
     public final DocumentParserContext addIgnoredFieldFromContext(IgnoredSourceFieldMapper.NameValue ignoredFieldWithNoSource)
         throws IOException {
         if (canAddIgnoredField()) {
-            if (inArrayScope) {
+            if (currentScope == Scope.ARRAY) {
                 // The field is an array within an array, store all sub-array elements.
                 ignoredFieldsMissingValues.add(ignoredFieldWithNoSource);
                 return cloneWithRecordedSource();
@@ -371,13 +386,14 @@ public final Collection<IgnoredSourceFieldMapper.NameValue> getIgnoredFieldsMiss
      * Applies to synthetic source only.
      */
     public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException {
-        if (canAddIgnoredField() && mapper instanceof ObjectMapper) {
-            boolean isNested = mapper instanceof NestedObjectMapper;
-            if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) {
-                DocumentParserContext subcontext = switchParser(parser());
-                subcontext.inArrayScope = inArrayScope == false;
-                return subcontext;
-            }
+        if (canAddIgnoredField()
+            && mapper instanceof ObjectMapper
+            && mapper instanceof NestedObjectMapper == false
+            && currentScope != Scope.ARRAY
+            && inArrayScopeEnabled) {
+            DocumentParserContext subcontext = switchParser(parser());
+            subcontext.currentScope = Scope.ARRAY;
+            return subcontext;
         }
         return this;
     }
@@ -667,6 +683,10 @@ public boolean isWithinCopyTo() {
         return false;
     }
 
+    public boolean inNestedScope() {
+        return currentScope == Scope.NESTED;
+    }
+
     public final DocumentParserContext createChildContext(ObjectMapper parent) {
         return new Wrapper(parent, this);
     }
@@ -704,12 +724,19 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested
      * Return a new context that has the provided document as the current document.
      */
     public final DocumentParserContext switchDoc(final LuceneDocument document) {
-        return new Wrapper(this.parent, this) {
+        DocumentParserContext cloned = new Wrapper(this.parent, this) {
             @Override
             public LuceneDocument doc() {
                 return document;
             }
         };
+
+        cloned.currentScope = Scope.NESTED;
+        // Disable using second parsing pass since it currently can not determine which parts
+        // of source belong to which nested document.
+        // See #115261.
+        cloned.inArrayScopeEnabled = false;
+        return cloned;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
index 4b6419b85e155..0793dd748c67e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
@@ -334,13 +334,10 @@ public boolean newDynamicStringField(DocumentParserContext context, String name)
                 );
             } else {
                 return createDynamicField(
-                    new TextFieldMapper.Builder(
-                        name,
-                        context.indexAnalyzers(),
-                        context.indexSettings().getMode().isSyntheticSourceEnabled()
-                    ).addMultiField(
-                        new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256)
-                    ),
+                    new TextFieldMapper.Builder(name, context.indexAnalyzers(), SourceFieldMapper.isSynthetic(context.indexSettings()))
+                        .addMultiField(
+                            new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256)
+                        ),
                     context
                 );
             }
@@ -412,10 +409,7 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D
         }
 
         boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException {
-            return createDynamicField(
-                new BinaryFieldMapper.Builder(name, context.indexSettings().getMode().isSyntheticSourceEnabled()),
-                context
-            );
+            return createDynamicField(new BinaryFieldMapper.Builder(name, SourceFieldMapper.isSynthetic(context.indexSettings())), context);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
index b9d89462c3467..8e418f45ddb3a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
@@ -22,6 +22,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -97,13 +98,13 @@ public boolean isSearchable() {
         @Override
         public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
             failIfNotIndexed();
-            BytesRef[] bytesRefs = values.stream().map(v -> {
+            List<BytesRef> bytesRefs = values.stream().map(v -> {
                 Object idObject = v;
                 if (idObject instanceof BytesRef) {
                     idObject = ((BytesRef) idObject).utf8ToString();
                 }
                 return Uid.encodeId(idObject.toString());
-            }).toArray(BytesRef[]::new);
+            }).toList();
             return new TermInSetQuery(name(), bytesRefs);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
index 296c2c5311d9a..70d73fc2ffb9a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
@@ -58,6 +58,9 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
 
     static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source");
     static final NodeFeature DONT_EXPAND_DOTS_IN_IGNORED_SOURCE = new NodeFeature("mapper.ignored_source.dont_expand_dots");
+    static final NodeFeature ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS = new NodeFeature(
+        "mapper.ignored_source.always_store_object_arrays_in_nested"
+    );
 
     /*
         Setting to disable encoding and writing values for this field.
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtil.java
index 6900dcd773917..8114167c02486 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtil.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtil.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 
 import java.util.ArrayList;
@@ -76,8 +75,8 @@ static CompiledAutomaton buildIpPrefixAutomaton(String ipPrefix) {
         } else {
             result = Automata.makeAnyBinary();
         }
-        result = MinimizationOperations.minimize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
-        return new CompiledAutomaton(result, null, false, 0, true);
+        result = Operations.determinize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        return new CompiledAutomaton(result, false, false, true);
     }
 
     private static Automaton getIpv6Automaton(String ipPrefix) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 529ff19bfffd7..ecc708bc94614 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -32,7 +32,6 @@
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton.AUTOMATON_TYPE;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.lucene.Lucene;
@@ -491,7 +490,7 @@ public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
             if (isIndexed()) {
                 return super.termsQuery(values, context);
             } else {
-                BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new);
+                Collection<BytesRef> bytesRefs = values.stream().map(this::indexedValueForSearch).toList();
                 return SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs);
             }
         }
@@ -597,7 +596,6 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi
                 ? AutomatonQueries.caseInsensitivePrefix(prefix)
                 : Operations.concatenate(Automata.makeString(prefix), Automata.makeAnyString());
             assert a.isDeterministic();
-            a = MinimizationOperations.minimize(a, 0);
 
             CompiledAutomaton automaton = new CompiledAutomaton(a, true, true);
 
@@ -632,14 +630,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (hasDocValues()) {
                 return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
             }
-            if (isSyntheticSource) {
-                if (false == isStored()) {
-                    throw new IllegalStateException(
-                        "keyword field ["
-                            + name()
-                            + "] is only supported in synthetic _source index if it creates doc values or stored fields"
-                    );
-                }
+            if (isStored()) {
                 return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
             }
             SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java
index f1924fd04f3fe..c6f1b490a2be2 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java
@@ -11,7 +11,6 @@
 
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -70,7 +69,7 @@ public Query termQuery(Object value, SearchExecutionContext context) {
 
         @Override
         public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
-            BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new);
+            var bytesRefs = values.stream().map(this::indexedValueForSearch).toList();
             return SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
index 4f90bd6e6f2c9..026c7c98d7aeb 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -52,12 +52,18 @@ public Set<NodeFeature> getFeatures() {
             IndexSettings.IGNORE_ABOVE_INDEX_LEVEL_SETTING,
             SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX,
             TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF,
-            FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT
+            FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT,
+            DenseVectorFieldMapper.BBQ_FORMAT
         );
     }
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
-        return Set.of(RangeFieldMapper.DATE_RANGE_INDEXING_FIX, IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE);
+        return Set.of(
+            RangeFieldMapper.DATE_RANGE_INDEXING_FIX,
+            IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE,
+            SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION,
+            IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java
index 6737f08b1ac5b..f30a0089e4eff 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -125,7 +124,10 @@ Mapping parse(@Nullable String type, MergeReason reason, Map<String, Object> map
         Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get();
         Map<String, Object> meta = null;
 
-        boolean isSourceSynthetic = mappingParserContext.getIndexSettings().getMode().isSyntheticSourceEnabled();
+        // TODO this should be the final value once `_source.mode` mapping parameter is not used anymore
+        // and it should not be reassigned below.
+        // For now it is still possible to set `_source.mode` so this is correct.
+        boolean isSourceSynthetic = SourceFieldMapper.isSynthetic(mappingParserContext.getIndexSettings());
         boolean isDataStream = false;
 
         Iterator<Map.Entry<String, Object>> iterator = mappingSource.entrySet().iterator();
@@ -147,10 +149,6 @@ Mapping parse(@Nullable String type, MergeReason reason, Map<String, Object> map
                 assert fieldNodeMap.isEmpty();
 
                 if (metadataFieldMapper instanceof SourceFieldMapper sfm) {
-                    // Validation in other places should have failed first
-                    assert sfm.isSynthetic()
-                        || (sfm.isSynthetic() == false && mappingParserContext.getIndexSettings().getMode() != IndexMode.TIME_SERIES)
-                        : "synthetic source can't be disabled in a time series index";
                     isSourceSynthetic = sfm.isSynthetic();
                 }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
index 8cc67cc481b9b..55ed1e10428aa 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -11,6 +11,7 @@
 
 import org.apache.lucene.document.DoubleField;
 import org.apache.lucene.document.DoublePoint;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatField;
 import org.apache.lucene.document.FloatPoint;
 import org.apache.lucene.document.IntField;
@@ -585,7 +586,7 @@ public Query rangeQuery(
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final float f = value.floatValue();
                 if (indexed && docValued) {
-                    document.add(new FloatField(name, f));
+                    document.add(new FloatField(name, f, Field.Store.NO));
                 } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f)));
                 } else if (indexed) {
@@ -735,7 +736,7 @@ public Query rangeQuery(
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final double d = value.doubleValue();
                 if (indexed && docValued) {
-                    document.add(new DoubleField(name, d));
+                    document.add(new DoubleField(name, d, Field.Store.NO));
                 } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d)));
                 } else if (indexed) {
@@ -1159,7 +1160,7 @@ public Query rangeQuery(
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final int i = value.intValue();
                 if (indexed && docValued) {
-                    document.add(new IntField(name, i));
+                    document.add(new IntField(name, i, Field.Store.NO));
                 } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, i));
                 } else if (indexed) {
@@ -1306,7 +1307,7 @@ public Query rangeQuery(
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final long l = value.longValue();
                 if (indexed && docValued) {
-                    document.add(new LongField(name, l));
+                    document.add(new LongField(name, l, Field.Store.NO));
                 } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, l));
                 } else if (indexed) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index 5e63fee8c5adc..70c4a3ac213a2 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.NodeFeature;
@@ -41,6 +42,7 @@
 
 public class ObjectMapper extends Mapper {
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class);
+    public static final FeatureFlag SUB_OBJECTS_AUTO_FEATURE_FLAG = new FeatureFlag("sub_objects_auto");
 
     public static final String CONTENT_TYPE = "object";
     static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source";
@@ -74,7 +76,7 @@ public static Subobjects from(Object node) {
                 if (value.equalsIgnoreCase("false")) {
                     return DISABLED;
                 }
-                if (value.equalsIgnoreCase("auto")) {
+                if (SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled() && value.equalsIgnoreCase("auto")) {
                     return AUTO;
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
index 80f845d626a2f..decc6d40a2f8e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
@@ -34,9 +34,6 @@
  * In case different pass-through objects contain subfields with the same name (excluding the pass-through prefix), their aliases conflict.
  * To resolve this, the pass-through spec specifies which object takes precedence through required parameter "priority"; non-negative
  * integer values are accepted, with the highest priority value winning in case of conflicting aliases.
- *
- * Note that this is an experimental, undocumented mapper type, currently intended for prototyping purposes only.
- * It has not been vetted for use in production systems.
  */
 public class PassThroughObjectMapper extends ObjectMapper {
     public static final String CONTENT_TYPE = "passthrough";
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index 118cdbffc5db9..372e0bbdfecf4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -18,11 +18,13 @@
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -37,8 +39,6 @@
 import java.util.List;
 import java.util.Locale;
 
-import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING;
-
 public class SourceFieldMapper extends MetadataFieldMapper {
     public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback");
     public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature(
@@ -51,6 +51,9 @@ public class SourceFieldMapper extends MetadataFieldMapper {
     public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature(
         "mapper.source.synthetic_source_copy_to_inside_objects_fix"
     );
+    public static final NodeFeature REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION = new NodeFeature(
+        "mapper.source.remove_synthetic_source_only_validation"
+    );
 
     public static final String NAME = "_source";
     public static final String RECOVERY_SOURCE_NAME = "_recovery_source";
@@ -59,8 +62,13 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
     public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters";
 
+    public static final Setting<Mode> INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> {
+        final IndexMode indexMode = IndexSettings.MODE.get(settings);
+        return indexMode.defaultSourceMode().name();
+    }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope);
+
     /** The source mode */
-    private enum Mode {
+    public enum Mode {
         DISABLED,
         STORED,
         SYNTHETIC
@@ -70,76 +78,28 @@ private enum Mode {
         null,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        null,
-        true
-    );
-
-    private static final SourceFieldMapper DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
-        null,
-        Explicit.IMPLICIT_TRUE,
-        Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        null,
-        false
-    );
-
-    private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper(
-        Mode.SYNTHETIC,
-        Explicit.IMPLICIT_TRUE,
-        Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        IndexMode.TIME_SERIES,
-        true
-    );
-
-    private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
-        Mode.SYNTHETIC,
-        Explicit.IMPLICIT_TRUE,
-        Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        IndexMode.TIME_SERIES,
-        false
+        Strings.EMPTY_ARRAY
     );
 
-    private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper(
-        Mode.SYNTHETIC,
+    private static final SourceFieldMapper STORED = new SourceFieldMapper(
+        Mode.STORED,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        IndexMode.LOGSDB,
-        true
+        Strings.EMPTY_ARRAY
     );
 
-    private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
+    private static final SourceFieldMapper SYNTHETIC = new SourceFieldMapper(
         Mode.SYNTHETIC,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        IndexMode.LOGSDB,
-        false
+        Strings.EMPTY_ARRAY
     );
 
-    /*
-     * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
-     * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
-     */
-    private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper(
-        null,
+    private static final SourceFieldMapper DISABLED = new SourceFieldMapper(
+        Mode.DISABLED,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        IndexMode.TIME_SERIES,
-        true
-    );
-
-    private static final SourceFieldMapper TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
-        null,
-        Explicit.IMPLICIT_TRUE,
-        Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        IndexMode.TIME_SERIES,
-        false
+        Strings.EMPTY_ARRAY
     );
 
     public static class Defaults {
@@ -194,23 +154,18 @@ public static class Builder extends MetadataFieldMapper.Builder {
             m -> Arrays.asList(toType(m).excludes)
         );
 
+        private final Settings settings;
+
         private final IndexMode indexMode;
 
         private final boolean supportsNonDefaultParameterValues;
 
-        private final boolean enableRecoverySource;
-
-        public Builder(
-            IndexMode indexMode,
-            final Settings settings,
-            boolean supportsCheckForNonDefaultParams,
-            boolean enableRecoverySource
-        ) {
+        public Builder(IndexMode indexMode, final Settings settings, boolean supportsCheckForNonDefaultParams) {
             super(Defaults.NAME);
+            this.settings = settings;
             this.indexMode = indexMode;
             this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false
                 || settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true);
-            this.enableRecoverySource = enableRecoverySource;
         }
 
         public Builder setSynthetic() {
@@ -224,31 +179,19 @@ protected Parameter<?>[] getParameters() {
         }
 
         private boolean isDefault() {
-            Mode m = mode.get();
-            if (m != null
-                && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
-                return false;
-            }
             return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
         }
 
         @Override
         public SourceFieldMapper build() {
             if (enabled.getValue().explicit()) {
-                if (indexMode != null && indexMode.isSyntheticSourceEnabled()) {
-                    throw new MapperParsingException("Indices with with index mode [" + indexMode + "] only support synthetic source");
-                }
                 if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }
             }
-            if (isDefault()) {
-                return switch (indexMode) {
-                    case TIME_SERIES -> enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE;
-                    case LOGSDB -> enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
-                    default -> enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
-                };
-            }
+
+            final Mode sourceMode = resolveSourceMode();
+
             if (supportsNonDefaultParameterValues == false) {
                 List<String> disallowed = new ArrayList<>();
                 if (enabled.get().value() == false) {
@@ -271,43 +214,81 @@ public SourceFieldMapper build() {
                     );
                 }
             }
-            SourceFieldMapper sourceFieldMapper = new SourceFieldMapper(
-                mode.get(),
-                enabled.get(),
-                includes.getValue().toArray(Strings.EMPTY_ARRAY),
-                excludes.getValue().toArray(Strings.EMPTY_ARRAY),
-                indexMode,
-                enableRecoverySource
-            );
+
+            if (sourceMode == Mode.SYNTHETIC && (includes.getValue().isEmpty() == false || excludes.getValue().isEmpty() == false)) {
+                throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source");
+            }
+
+            SourceFieldMapper sourceFieldMapper;
+            if (isDefault()) {
+                // Needed for bwc so that "mode" is not serialized in case of a standard index with stored source.
+                if (sourceMode == null) {
+                    sourceFieldMapper = DEFAULT;
+                } else {
+                    sourceFieldMapper = resolveStaticInstance(sourceMode);
+                }
+            } else {
+                sourceFieldMapper = new SourceFieldMapper(
+                    sourceMode,
+                    enabled.get(),
+                    includes.getValue().toArray(Strings.EMPTY_ARRAY),
+                    excludes.getValue().toArray(Strings.EMPTY_ARRAY)
+                );
+            }
             if (indexMode != null) {
                 indexMode.validateSourceFieldMapper(sourceFieldMapper);
             }
             return sourceFieldMapper;
         }
 
-    }
+        private Mode resolveSourceMode() {
+            // If the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source`
+            // otherwise the mode is determined according to `_source.mode`.
+            if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) {
+                return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings);
+            }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
-        var indexMode = c.getIndexSettings().getMode();
-        boolean enableRecoverySource = INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings());
-        if (indexMode.isSyntheticSourceEnabled()) {
-            if (indexMode == IndexMode.TIME_SERIES) {
-                if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) {
-                    return enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE;
-                } else {
-                    return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE;
+            // If `_source.mode` is not set we need to apply a default according to index mode.
+            if (mode.get() == null) {
+                if (indexMode == null || indexMode == IndexMode.STANDARD) {
+                    // Special case to avoid serializing mode.
+                    return null;
                 }
-            } else if (indexMode == IndexMode.LOGSDB) {
-                return enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
+
+                return indexMode.defaultSourceMode();
             }
+
+            return mode.get();
+        }
+    }
+
+    private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) {
+        return switch (sourceMode) {
+            case SYNTHETIC -> SYNTHETIC;
+            case STORED -> STORED;
+            case DISABLED -> DISABLED;
+        };
+    }
+
+    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
+        final IndexMode indexMode = c.getIndexSettings().getMode();
+
+        if (indexMode == IndexMode.TIME_SERIES && c.getIndexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)) {
+            return DEFAULT;
+        }
+
+        final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings());
+        // Needed for bwc so that "mode" is not serialized in case of standard index with stored source.
+        if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) {
+            return DEFAULT;
         }
-        return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
+
+        return resolveStaticInstance(settingSourceMode);
     },
         c -> new Builder(
             c.getIndexSettings().getMode(),
             c.getSettings(),
-            c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK),
-            INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings())
+            c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK)
         )
     );
 
@@ -359,30 +340,14 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
     private final String[] excludes;
     private final SourceFilter sourceFilter;
 
-    private final IndexMode indexMode;
-    private final boolean enableRecoverySource;
-
-    private SourceFieldMapper(
-        Mode mode,
-        Explicit<Boolean> enabled,
-        String[] includes,
-        String[] excludes,
-        IndexMode indexMode,
-        boolean enableRecoverySource
-    ) {
+    private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] includes, String[] excludes) {
         super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED)));
-        assert enabled.explicit() == false || mode == null;
         this.mode = mode;
         this.enabled = enabled;
         this.sourceFilter = buildSourceFilter(includes, excludes);
         this.includes = includes;
         this.excludes = excludes;
-        if (this.sourceFilter != null && (mode == Mode.SYNTHETIC || indexMode == IndexMode.TIME_SERIES)) {
-            throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source");
-        }
         this.complete = stored() && sourceFilter == null;
-        this.indexMode = indexMode;
-        this.enableRecoverySource = enableRecoverySource;
     }
 
     private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) {
@@ -420,13 +385,11 @@ public void preParse(DocumentParserContext context) throws IOException {
         final BytesReference adaptedSource = applyFilters(originalSource, contentType);
 
         if (adaptedSource != null) {
-            assert context.indexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)
-                || indexMode == null
-                || indexMode.isSyntheticSourceEnabled() == false;
             final BytesRef ref = adaptedSource.toBytesRef();
             context.doc().add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length));
         }
 
+        boolean enableRecoverySource = context.indexSettings().isRecoverySourceEnabled();
         if (enableRecoverySource && originalSource != null && adaptedSource != originalSource) {
             // if we omitted source or modified it we add the _recovery_source to ensure we have it for ops based recovery
             BytesRef ref = originalSource.toBytesRef();
@@ -455,7 +418,7 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(indexMode, Settings.EMPTY, false, enableRecoverySource).init(this);
+        return new Builder(null, Settings.EMPTY, false).init(this);
     }
 
     /**
@@ -471,4 +434,16 @@ public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics)
     public boolean isSynthetic() {
         return mode == Mode.SYNTHETIC;
     }
+
+    public static boolean isSynthetic(IndexSettings indexSettings) {
+        return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC;
+    }
+
+    public boolean isDisabled() {
+        return mode == Mode.DISABLED;
+    }
+
+    public boolean isStored() {
+        return mode == null || mode == Mode.STORED;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index 9ea16933f7ab5..ceb96b87a0983 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -101,9 +101,7 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool
         failIfNotIndexed();
         Term prefix = new Term(name(), indexedValueForSearch(value));
         if (caseInsensitive) {
-            return method == null
-                ? new CaseInsensitivePrefixQuery(prefix, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false)
-                : new CaseInsensitivePrefixQuery(prefix, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false, method);
+            return method == null ? new CaseInsensitivePrefixQuery(prefix, false) : new CaseInsensitivePrefixQuery(prefix, false, method);
         }
         return method == null ? new PrefixQuery(prefix) : new PrefixQuery(prefix, method);
     }
@@ -170,9 +168,7 @@ protected Query wildcardQuery(
             term = new Term(name(), indexedValueForSearch(value));
         }
         if (caseInsensitive) {
-            return method == null
-                ? new CaseInsensitiveWildcardQuery(term)
-                : new CaseInsensitiveWildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false, method);
+            return method == null ? new CaseInsensitiveWildcardQuery(term) : new CaseInsensitiveWildcardQuery(term, false, method);
         }
         return method == null ? new WildcardQuery(term) : new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, method);
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
index 674a016264c3a..e2ff9cc7ea632 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 
 /** Base {@link MappedFieldType} implementation for a field that is indexed
@@ -69,7 +70,7 @@ public Query termQuery(Object value, SearchExecutionContext context) {
     @Override
     public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
         failIfNotIndexed();
-        BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new);
+        List<BytesRef> bytesRefs = values.stream().map(this::indexedValueForSearch).toList();
         return new TermInSetQuery(name(), bytesRefs);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index 2c55fc35db57d..253f70f4fda47 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -239,7 +239,7 @@ public static class Builder extends FieldMapper.Builder {
         private final IndexVersion indexCreatedVersion;
         private final Parameter<Boolean> store;
 
-        private final boolean isSyntheticSourceEnabledViaIndexMode;
+        private final boolean isSyntheticSourceEnabled;
 
         private final Parameter<Boolean> index = Parameter.indexParam(m -> ((TextFieldMapper) m).index, true);
 
@@ -286,16 +286,11 @@ public static class Builder extends FieldMapper.Builder {
 
         final TextParams.Analyzers analyzers;
 
-        public Builder(String name, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabledViaIndexMode) {
-            this(name, IndexVersion.current(), indexAnalyzers, isSyntheticSourceEnabledViaIndexMode);
+        public Builder(String name, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) {
+            this(name, IndexVersion.current(), indexAnalyzers, isSyntheticSourceEnabled);
         }
 
-        public Builder(
-            String name,
-            IndexVersion indexCreatedVersion,
-            IndexAnalyzers indexAnalyzers,
-            boolean isSyntheticSourceEnabledViaIndexMode
-        ) {
+        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) {
             super(name);
 
             // If synthetic source is used we need to either store this field
@@ -306,7 +301,7 @@ public Builder(
             // If 'store' parameter was explicitly provided we'll reject the request.
             this.store = Parameter.storeParam(
                 m -> ((TextFieldMapper) m).store,
-                () -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false
+                () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false
             );
             this.indexCreatedVersion = indexCreatedVersion;
             this.analyzers = new TextParams.Analyzers(
@@ -315,7 +310,7 @@ public Builder(
                 m -> (((TextFieldMapper) m).positionIncrementGap),
                 indexCreatedVersion
             );
-            this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode;
+            this.isSyntheticSourceEnabled = isSyntheticSourceEnabled;
         }
 
         public Builder index(boolean index) {
@@ -488,7 +483,7 @@ public TextFieldMapper build(MapperBuilderContext context) {
     private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
 
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled()),
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings())),
         MINIMUM_COMPATIBILITY_VERSION
     );
 
@@ -603,8 +598,8 @@ public Query prefixQuery(
             }
             Automaton automaton = Operations.concatenate(automata);
             AutomatonQuery query = method == null
-                ? new AutomatonQuery(new Term(name(), value + "*"), automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false)
-                : new AutomatonQuery(new Term(name(), value + "*"), automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false, method);
+                ? new AutomatonQuery(new Term(name(), value + "*"), automaton, false)
+                : new AutomatonQuery(new Term(name(), value + "*"), automaton, false, method);
             return new BooleanQuery.Builder().add(query, BooleanClause.Occur.SHOULD)
                 .add(new TermQuery(new Term(parentField.name(), value)), BooleanClause.Occur.SHOULD)
                 .build();
@@ -1012,15 +1007,6 @@ protected String delegatingTo() {
             if (isStored()) {
                 return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name());
             }
-            if (isSyntheticSource) {
-                /*
-                 * When we're in synthetic source mode we don't currently
-                 * support text fields that are not stored and are not children
-                 * of perfect keyword fields. We'd have to load from the parent
-                 * field and then convert the result to a string.
-                 */
-                return null;
-            }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext));
         }
@@ -1239,7 +1225,7 @@ public Query existsQuery(SearchExecutionContext context) {
     private final SubFieldInfo prefixFieldInfo;
     private final SubFieldInfo phraseFieldInfo;
 
-    private final boolean isSyntheticSourceEnabledViaIndexMode;
+    private final boolean isSyntheticSourceEnabled;
 
     private TextFieldMapper(
         String simpleName,
@@ -1272,7 +1258,7 @@ private TextFieldMapper(
         this.indexPrefixes = builder.indexPrefixes.getValue();
         this.freqFilter = builder.freqFilter.getValue();
         this.fieldData = builder.fieldData.get();
-        this.isSyntheticSourceEnabledViaIndexMode = builder.isSyntheticSourceEnabledViaIndexMode;
+        this.isSyntheticSourceEnabled = builder.isSyntheticSourceEnabled;
     }
 
     @Override
@@ -1296,7 +1282,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabledViaIndexMode).init(this);
+        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabled).init(this);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java
index ac1de94ea7a73..93a2157b2338a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java
@@ -28,10 +28,10 @@
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOBooleanSupplier;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
@@ -394,7 +394,6 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi
                 a = Operations.concatenate(a, Automata.makeAnyString());
             }
             assert a.isDeterministic();
-            a = MinimizationOperations.minimize(a, 0);
 
             CompiledAutomaton automaton = new CompiledAutomaton(a);
             if (searchAfter != null) {
@@ -483,6 +482,11 @@ public AttributeSource attributes() {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public IOBooleanSupplier prepareSeekExact(BytesRef bytesRef) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public boolean seekExact(BytesRef text) throws IOException {
             throw new UnsupportedOperationException();
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java
index b94ea67c8de8d..b29f093e3a217 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java
@@ -205,12 +205,8 @@ public long nextOrd() throws IOException {
             }
 
             long ord = delegate.nextOrd();
-            if (ord != NO_MORE_ORDS && ord <= maxOrd) {
-                assert ord >= minOrd;
-                return mapOrd(ord);
-            } else {
-                return NO_MORE_ORDS;
-            }
+            assert ord <= maxOrd;
+            return mapOrd(ord);
         }
 
         @Override
@@ -223,9 +219,9 @@ public boolean advanceExact(int target) throws IOException {
             if (delegate.advanceExact(target)) {
 
                 int count = 0;
-                while (true) {
+                for (int i = 0; i < delegate.docValueCount(); i++) {
                     long ord = delegate.nextOrd();
-                    if (ord == NO_MORE_ORDS || ord > maxOrd) {
+                    if (ord > maxOrd) {
                         break;
                     }
                     if (ord >= minOrd) {
@@ -246,7 +242,7 @@ public boolean advanceExact(int target) throws IOException {
 
                 while (true) {
                     long ord = delegate.nextOrd();
-                    if (ord == NO_MORE_ORDS || ord > maxOrd) {
+                    if (ord > maxOrd) {
                         break;
                     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValues.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValues.java
index e8da3b72ae7c7..04069333deb13 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValues.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValues.java
@@ -45,24 +45,13 @@ public int size() {
     }
 
     @Override
-    public float[] vectorValue() throws IOException {
-        // Lazy load vectors as we may iterate but not actually require the vector
-        return vectorValue(in.docID());
+    public DocIndexIterator iterator() {
+        return in.iterator();
     }
 
     @Override
-    public int docID() {
-        return in.docID();
-    }
-
-    @Override
-    public int nextDoc() throws IOException {
-        return in.nextDoc();
-    }
-
-    @Override
-    public int advance(int target) throws IOException {
-        return in.advance(target);
+    public FloatVectorValues copy() throws IOException {
+        return in.copy();
     }
 
     @Override
@@ -74,22 +63,24 @@ public float magnitude() {
         return magnitude;
     }
 
-    private float[] vectorValue(int docId) throws IOException {
+    @Override
+    public float[] vectorValue(int ord) throws IOException {
+        int docId = ordToDoc(ord);
         if (docId != this.docId) {
             this.docId = docId;
             hasMagnitude = decodedMagnitude(docId);
             // We should only copy and transform if we have a stored a non-unit length magnitude
             if (hasMagnitude) {
-                System.arraycopy(in.vectorValue(), 0, vector, 0, dimension());
+                System.arraycopy(in.vectorValue(ord), 0, vector, 0, dimension());
                 for (int i = 0; i < vector.length; i++) {
                     vector[i] *= magnitude;
                 }
                 return vector;
             } else {
-                return in.vectorValue();
+                return in.vectorValue(ord);
             }
         } else {
-            return hasMagnitude ? vector : in.vectorValue();
+            return hasMagnitude ? vector : in.vectorValue(ord);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
index d7353584706d8..809532c0e8f5a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SegmentReadState;
@@ -45,6 +46,8 @@
 import org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat;
 import org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat;
 import org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat;
+import org.elasticsearch.index.codec.vectors.ES816BinaryQuantizedVectorsFormat;
+import org.elasticsearch.index.codec.vectors.ES816HnswBinaryQuantizedVectorsFormat;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.ArraySourceValueFetcher;
@@ -98,6 +101,7 @@
 public class DenseVectorFieldMapper extends FieldMapper {
     public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude";
     private static final float EPS = 1e-3f;
+    static final int BBQ_MIN_DIMS = 64;
 
     public static boolean isNotUnitVector(float magnitude) {
         return Math.abs(magnitude - 1.0f) > EPS;
@@ -105,6 +109,7 @@ public static boolean isNotUnitVector(float magnitude) {
 
     public static final NodeFeature INT4_QUANTIZATION = new NodeFeature("mapper.vectors.int4_quantization");
     public static final NodeFeature BIT_VECTORS = new NodeFeature("mapper.vectors.bit_vectors");
+    public static final NodeFeature BBQ_FORMAT = new NodeFeature("mapper.vectors.bbq");
 
     public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersions.V_7_5_0;
     public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION;
@@ -1162,7 +1167,7 @@ final void validateElementType(ElementType elementType) {
 
         abstract boolean updatableTo(IndexOptions update);
 
-        public final void validateDimension(int dim) {
+        public void validateDimension(int dim) {
             if (type.supportsDimension(dim)) {
                 return;
             }
@@ -1342,6 +1347,50 @@ public boolean supportsElementType(ElementType elementType) {
             public boolean supportsDimension(int dims) {
                 return dims % 2 == 0;
             }
+        },
+        BBQ_HNSW("bbq_hnsw") {
+            @Override
+            public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap) {
+                Object mNode = indexOptionsMap.remove("m");
+                Object efConstructionNode = indexOptionsMap.remove("ef_construction");
+                if (mNode == null) {
+                    mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
+                }
+                if (efConstructionNode == null) {
+                    efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+                }
+                int m = XContentMapValues.nodeIntegerValue(mNode);
+                int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode);
+                MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
+                return new BBQHnswIndexOptions(m, efConstruction);
+            }
+
+            @Override
+            public boolean supportsElementType(ElementType elementType) {
+                return elementType == ElementType.FLOAT;
+            }
+
+            @Override
+            public boolean supportsDimension(int dims) {
+                return dims >= BBQ_MIN_DIMS;
+            }
+        },
+        BBQ_FLAT("bbq_flat") {
+            @Override
+            public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap) {
+                MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
+                return new BBQFlatIndexOptions();
+            }
+
+            @Override
+            public boolean supportsElementType(ElementType elementType) {
+                return elementType == ElementType.FLOAT;
+            }
+
+            @Override
+            public boolean supportsDimension(int dims) {
+                return dims >= BBQ_MIN_DIMS;
+            }
         };
 
         static Optional<VectorIndexType> fromString(String type) {
@@ -1707,6 +1756,102 @@ public String toString() {
         }
     }
 
+    static class BBQHnswIndexOptions extends IndexOptions {
+        private final int m;
+        private final int efConstruction;
+
+        BBQHnswIndexOptions(int m, int efConstruction) {
+            super(VectorIndexType.BBQ_HNSW);
+            this.m = m;
+            this.efConstruction = efConstruction;
+        }
+
+        @Override
+        KnnVectorsFormat getVectorsFormat(ElementType elementType) {
+            assert elementType == ElementType.FLOAT;
+            return new ES816HnswBinaryQuantizedVectorsFormat(m, efConstruction);
+        }
+
+        @Override
+        boolean updatableTo(IndexOptions update) {
+            return update.type.equals(this.type);
+        }
+
+        @Override
+        boolean doEquals(IndexOptions other) {
+            BBQHnswIndexOptions that = (BBQHnswIndexOptions) other;
+            return m == that.m && efConstruction == that.efConstruction;
+        }
+
+        @Override
+        int doHashCode() {
+            return Objects.hash(m, efConstruction);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("type", type);
+            builder.field("m", m);
+            builder.field("ef_construction", efConstruction);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void validateDimension(int dim) {
+            if (type.supportsDimension(dim)) {
+                return;
+            }
+            throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
+        }
+    }
+
+    static class BBQFlatIndexOptions extends IndexOptions {
+        private final int CLASS_NAME_HASH = this.getClass().getName().hashCode();
+
+        BBQFlatIndexOptions() {
+            super(VectorIndexType.BBQ_FLAT);
+        }
+
+        @Override
+        KnnVectorsFormat getVectorsFormat(ElementType elementType) {
+            assert elementType == ElementType.FLOAT;
+            return new ES816BinaryQuantizedVectorsFormat();
+        }
+
+        @Override
+        boolean updatableTo(IndexOptions update) {
+            return update.type.equals(this.type);
+        }
+
+        @Override
+        boolean doEquals(IndexOptions other) {
+            return other instanceof BBQFlatIndexOptions;
+        }
+
+        @Override
+        int doHashCode() {
+            return CLASS_NAME_HASH;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("type", type);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void validateDimension(int dim) {
+            if (type.supportsDimension(dim)) {
+                return;
+            }
+            throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
+        }
+    }
+
     public static final TypeParser PARSER = new TypeParser(
         (n, c) -> new Builder(n, c.indexVersionCreated()),
         notInMultiFields(CONTENT_TYPE)
@@ -2108,9 +2253,12 @@ private static IndexOptions parseIndexOptions(String fieldName, Object propNode)
             throw new MapperParsingException("[index_options] requires field [type] to be configured");
         }
         String type = XContentMapValues.nodeStringValue(typeNode);
-        return VectorIndexType.fromString(type)
-            .orElseThrow(() -> new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]"))
-            .parseIndexOptions(fieldName, indexOptionsMap);
+        Optional<VectorIndexType> vectorIndexType = VectorIndexType.fromString(type);
+        if (vectorIndexType.isEmpty()) {
+            throw new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]");
+        }
+        VectorIndexType parsedType = vectorIndexType.get();
+        return parsedType.parseIndexOptions(fieldName, indexOptionsMap);
     }
 
     /**
@@ -2162,6 +2310,7 @@ private class IndexedSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyn
         private ByteVectorValues byteVectorValues;
         private boolean hasValue;
         private boolean hasMagnitude;
+        private int ord;
 
         private final IndexVersion indexCreatedVersion;
         private final VectorSimilarity vectorSimilarity;
@@ -2179,16 +2328,20 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf
                 if (indexCreatedVersion.onOrAfter(NORMALIZE_COSINE) && VectorSimilarity.COSINE.equals(vectorSimilarity)) {
                     magnitudeReader = leafReader.getNumericDocValues(fullPath() + COSINE_MAGNITUDE_FIELD_SUFFIX);
                 }
+                KnnVectorValues.DocIndexIterator iterator = values.iterator();
                 return docId -> {
-                    hasValue = docId == values.advance(docId);
+                    hasValue = docId == iterator.advance(docId);
                     hasMagnitude = hasValue && magnitudeReader != null && magnitudeReader.advanceExact(docId);
+                    ord = iterator.index();
                     return hasValue;
                 };
             }
             byteVectorValues = leafReader.getByteVectorValues(fullPath());
             if (byteVectorValues != null) {
+                KnnVectorValues.DocIndexIterator iterator = byteVectorValues.iterator();
                 return docId -> {
-                    hasValue = docId == byteVectorValues.advance(docId);
+                    hasValue = docId == iterator.advance(docId);
+                    ord = iterator.index();
                     return hasValue;
                 };
             }
@@ -2211,7 +2364,7 @@ public void write(XContentBuilder b) throws IOException {
             }
             b.startArray(leafName());
             if (values != null) {
-                for (float v : values.vectorValue()) {
+                for (float v : values.vectorValue(ord)) {
                     if (hasMagnitude) {
                         b.value(v * magnitude);
                     } else {
@@ -2219,7 +2372,7 @@ public void write(XContentBuilder b) throws IOException {
                     }
                 }
             } else if (byteVectorValues != null) {
-                byte[] vectorValue = byteVectorValues.vectorValue();
+                byte[] vectorValue = byteVectorValues.vectorValue(ord);
                 for (byte value : vectorValue) {
                     b.value(value);
                 }
@@ -2270,7 +2423,7 @@ public void write(XContentBuilder b) throws IOException {
             if (indexCreatedVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION)) {
                 byteBuffer.order(ByteOrder.LITTLE_ENDIAN);
             }
-            int dims = fieldType().dims;
+            int dims = fieldType().elementType == ElementType.BIT ? fieldType().dims / Byte.SIZE : fieldType().dims;
             for (int dim = 0; dim < dims; dim++) {
                 fieldType().elementType.readAndWriteValue(byteBuffer, b);
             }
diff --git a/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java b/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java
index df49e00f8af73..7c40fdc93a48b 100644
--- a/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java
+++ b/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java
@@ -50,4 +50,8 @@ public long getTotalBytesSize() {
     public List<SegmentCommitInfo> getMergedSegments() {
         return oneMerge.segments;
     }
+
+    public MergePolicy.OneMerge getMerge() {
+        return oneMerge;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
index 7549873a10bc1..033151da362ef 100644
--- a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.GeometryCollection;
 import org.elasticsearch.geometry.ShapeType;
@@ -429,9 +428,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             GeoJson.toXContent(shape, builder, params);
         } else {
             builder.startObject(INDEXED_SHAPE_FIELD.getPreferredName()).field(SHAPE_ID_FIELD.getPreferredName(), indexedShapeId);
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(SHAPE_TYPE_FIELD.getPreferredName(), MapperService.SINGLE_MAPPING_NAME);
-            }
             if (indexedShapeIndex != null) {
                 builder.field(SHAPE_INDEX_FIELD.getPreferredName(), indexedShapeIndex);
             }
@@ -555,16 +551,13 @@ public static ParsedGeometryQueryParams parsedParamsFromXContent(XContentParser
                                 } else if (token.isValue()) {
                                     if (SHAPE_ID_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                                         params.id = parser.text();
-                                    } else if (parser.getRestApiVersion() == RestApiVersion.V_7
-                                        && SHAPE_TYPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                                            deprecationLogger.compatibleCritical("geo_share_query_with_types", TYPES_DEPRECATION_MESSAGE);
-                                        } else if (SHAPE_INDEX_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                                            params.index = parser.text();
-                                        } else if (SHAPE_PATH_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                                            params.shapePath = parser.text();
-                                        } else if (SHAPE_ROUTING_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                                            params.shapeRouting = parser.text();
-                                        }
+                                    } else if (SHAPE_INDEX_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                                        params.index = parser.text();
+                                    } else if (SHAPE_PATH_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                                        params.shapePath = parser.text();
+                                    } else if (SHAPE_ROUTING_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                                        params.shapeRouting = parser.text();
+                                    }
                                 } else {
                                     throw new ParsingException(
                                         parser.getTokenLocation(),
diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index a7b3b9145d2ca..5329dbf01975a 100644
--- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -238,9 +237,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         doXArrayContent(FILTER, filterClauses, builder, params);
         doXArrayContent(MUST_NOT, mustNotClauses, builder, params);
         doXArrayContent(SHOULD, shouldClauses, builder, params);
-        if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-            builder.field(ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative);
-        } else if (adjustPureNegative != ADJUST_PURE_NEGATIVE_DEFAULT) {
+        if (adjustPureNegative != ADJUST_PURE_NEGATIVE_DEFAULT) {
             builder.field(ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative);
         }
         if (minimumShouldMatch != null) {
@@ -359,6 +356,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
         if (mustClauses.size() == 0
             && filterClauses.size() == 0
             && shouldClauses.size() > 0
+            && mustNotClauses.size() == 0
             && newBuilder.shouldClauses.stream().allMatch(b -> b instanceof MatchNoneQueryBuilder)) {
             return new MatchNoneQueryBuilder("The \"" + getName() + "\" query was rewritten to a \"match_none\" query.");
         }
diff --git a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java
index 16aada4066f71..1560004b13785 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java
@@ -412,8 +412,8 @@ public Query createPhraseQuery(String field, String queryText, int phraseSlop) {
         protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
             CombinedFieldQuery.Builder query = new CombinedFieldQuery.Builder();
             for (TermAndBoost termAndBoost : terms) {
-                assert termAndBoost.boost == BoostAttribute.DEFAULT_BOOST;
-                BytesRef bytes = termAndBoost.term;
+                assert termAndBoost.boost() == BoostAttribute.DEFAULT_BOOST;
+                BytesRef bytes = termAndBoost.term();
                 query.addTerm(bytes);
             }
             for (FieldAndBoost fieldAndBoost : fields) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
index 263c6bd35bcca..0b9663d9112fa 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -27,6 +28,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
     public static final String COMMON_TERMS_QUERY_DEPRECATION_MSG = "Common Terms Query usage is not supported. "
         + "Use [match] query which can efficiently skip blocks of documents if the total number of hits is not tracked.";
 
+    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_RELEVANCE) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
     public static ParseField NAME_V7 = new ParseField("common").withAllDeprecated(COMMON_TERMS_QUERY_DEPRECATION_MSG)
         .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java
index 3e5deeeebae5d..964358610e074 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java
@@ -9,17 +9,23 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.search.Query;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
 import java.util.Collections;
+import java.util.Map;
 import java.util.function.LongSupplier;
 
 /**
@@ -30,20 +36,57 @@
  * and skip the shards that don't hold queried data. See IndexMetadata for more details.
  */
 public class CoordinatorRewriteContext extends QueryRewriteContext {
+
+    public static final String TIER_FIELD_NAME = "_tier";
+
+    private static final ConstantFieldType TIER_FIELD_TYPE = new ConstantFieldType(TIER_FIELD_NAME, Map.of()) {
+        @Override
+        public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
+            throw new UnsupportedOperationException("fetching field values is not supported on the coordinator node");
+        }
+
+        @Override
+        public String typeName() {
+            return TIER_FIELD_NAME;
+        }
+
+        @Override
+        protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteContext context) {
+            if (caseInsensitive) {
+                pattern = Strings.toLowercaseAscii(pattern);
+            }
+
+            String tierPreference = context.getTierPreference();
+            if (tierPreference == null) {
+                return false;
+            }
+            return Regex.simpleMatch(pattern, tierPreference);
+        }
+
+        @Override
+        public Query existsQuery(SearchExecutionContext context) {
+            throw new UnsupportedOperationException("field exists query is not supported on the coordinator node");
+        }
+    };
+
     private final DateFieldRangeInfo dateFieldRangeInfo;
+    private final String tier;
 
     /**
      * Context for coordinator search rewrites based on time ranges for the @timestamp field and/or 'event.ingested' field
+     *
      * @param parserConfig
      * @param client
      * @param nowInMillis
      * @param dateFieldRangeInfo range and field type info for @timestamp and 'event.ingested'
+     * @param tier               the configured data tier (via the _tier_preference setting) for the index
      */
     public CoordinatorRewriteContext(
         XContentParserConfiguration parserConfig,
         Client client,
         LongSupplier nowInMillis,
-        DateFieldRangeInfo dateFieldRangeInfo
+        DateFieldRangeInfo dateFieldRangeInfo,
+        String tier
     ) {
         super(
             parserConfig,
@@ -63,10 +106,12 @@ public CoordinatorRewriteContext(
             null
         );
         this.dateFieldRangeInfo = dateFieldRangeInfo;
+        this.tier = tier;
     }
 
     /**
-     * @param fieldName Must be one of DataStream.TIMESTAMP_FIELD_FIELD or IndexMetadata.EVENT_INGESTED_FIELD_NAME
+     * @param fieldName Must be one of DataStream.TIMESTAMP_FIELD_FIELD, IndexMetadata.EVENT_INGESTED_FIELD_NAME, or
+     *                  DataTierFiledMapper.NAME
      * @return MappedField with type for the field. Returns null if fieldName is not one of the allowed field names.
      */
     @Nullable
@@ -75,6 +120,8 @@ public MappedFieldType getFieldType(String fieldName) {
             return dateFieldRangeInfo.timestampFieldType();
         } else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
             return dateFieldRangeInfo.eventIngestedFieldType();
+        } else if (TIER_FIELD_NAME.equals(fieldName)) {
+            return TIER_FIELD_TYPE;
         } else {
             return null;
         }
@@ -99,4 +146,18 @@ public IndexLongFieldRange getFieldRange(String fieldName) {
     public CoordinatorRewriteContext convertToCoordinatorRewriteContext() {
         return this;
     }
+
+    @Override
+    public String getTierPreference() {
+        // dominant branch first (tier preference is configured)
+        return tier.isEmpty() == false ? tier : null;
+    }
+
+    /**
+     * We're holding on to the index tier in the context as otherwise we'd need
+     * to re-parse it from the index settings when evaluating the _tier field.
+     */
+    public String tier() {
+        return tier;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java
index 67042a98db42a..e48d7699d03ef 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java
@@ -52,6 +52,12 @@ public CoordinatorRewriteContext getCoordinatorRewriteContext(Index index) {
             return null;
         }
         DateFieldRangeInfo dateFieldRangeInfo = mappingSupplier.apply(index);
+        // we've now added a coordinator rewrite based on the _tier field so the requirement
+        // for the timestamps fields to be present is artificial (we could do a coordinator
+        // rewrite only based on the _tier field) and we might decide to remove this artificial
+        // limitation to enable coordinator rewrites based on _tier for hot and warm indices
+        // (currently the _tier coordinator rewrite is only available for mounted and partially mounted
+        // indices)
         if (dateFieldRangeInfo == null) {
             return null;
         }
@@ -74,7 +80,8 @@ public CoordinatorRewriteContext getCoordinatorRewriteContext(Index index) {
             parserConfig,
             client,
             nowInMillis,
-            new DateFieldRangeInfo(timestampFieldType, timestampRange, dateFieldRangeInfo.eventIngestedFieldType(), eventIngestedRange)
+            new DateFieldRangeInfo(timestampFieldType, timestampRange, dateFieldRangeInfo.eventIngestedFieldType(), eventIngestedRange),
+            indexMetadata.getTierPreference().isEmpty() == false ? indexMetadata.getTierPreference().getFirst() : ""
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index 86413404d9571..7d773ce6c3fd2 100644
--- a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.geometry.utils.Geohash;
 import org.elasticsearch.index.mapper.GeoShapeQueryable;
@@ -55,7 +54,6 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
      */
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
-    private static final ParseField TYPE_FIELD = new ParseField("type").forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
     private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
 
@@ -358,18 +356,14 @@ public static GeoBoundingBoxQueryBuilder fromXContent(XContentParser parser) thr
                     validationMethod = GeoValidationMethod.fromString(parser.text());
                 } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parser.getRestApiVersion() == RestApiVersion.V_7
-                    && TYPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        deprecationLogger.compatibleCritical("geo_bounding_box_type", TYPE_PARAMETER_DEPRECATION_MESSAGE);
-                        parser.text(); // ignore value
-                    } else {
-                        throw new ParsingException(
-                            parser.getTokenLocation(),
-                            "failed to parse [{}] query. unexpected field [{}]",
-                            NAME,
-                            currentFieldName
-                        );
-                    }
+                } else {
+                    throw new ParsingException(
+                        parser.getTokenLocation(),
+                        "failed to parse [{}] query. unexpected field [{}]",
+                        NAME,
+                        currentFieldName
+                    );
+                }
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java
index f21edaeb94f22..b2b37ad834178 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java
@@ -20,7 +20,7 @@
 import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings;
@@ -189,7 +189,7 @@ protected List<IntervalsSource> analyzeGraph(TokenStream source) throws IOExcept
         List<IntervalsSource> clauses = new ArrayList<>();
         int[] articulationPoints = graph.articulationPoints();
         int lastState = 0;
-        int maxClauseCount = BooleanQuery.getMaxClauseCount();
+        int maxClauseCount = IndexSearcher.getMaxClauseCount();
         for (int i = 0; i <= articulationPoints.length; i++) {
             int start = lastState;
             int end = -1;
@@ -204,7 +204,7 @@ protected List<IntervalsSource> analyzeGraph(TokenStream source) throws IOExcept
                     TokenStream ts = it.next();
                     IntervalsSource phrase = combineSources(analyzeTerms(ts), 0, true);
                     if (paths.size() >= maxClauseCount) {
-                        throw new BooleanQuery.TooManyClauses();
+                        throw new IndexSearcher.TooManyClauses();
                     }
                     paths.add(phrase);
                 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
index a4a76c078cb55..fd704d39ca384 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.support.QueryParsers;
@@ -37,11 +36,7 @@
  * result of the analysis.
  */
 public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
-    private static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "cutoff_freqency is not supported. "
-        + "The [match] query can skip block of documents efficiently if the total number of hits is not tracked";
-    public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency").withAllDeprecated(
-        CUTOFF_FREQUENCY_DEPRECATION_MSG
-    ).forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
+
     public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query");
     public static final ParseField LENIENT_FIELD = new ParseField("lenient");
     public static final ParseField FUZZY_TRANSPOSITIONS_FIELD = new ParseField("fuzzy_transpositions");
@@ -528,15 +523,12 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc
                             queryName = parser.text();
                         } else if (GENERATE_SYNONYMS_PHRASE_QUERY.match(currentFieldName, parser.getDeprecationHandler())) {
                             autoGenerateSynonymsPhraseQuery = parser.booleanValue();
-                        } else if (parser.getRestApiVersion() == RestApiVersion.V_7
-                            && CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                                throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG);
-                            } else {
-                                throw new ParsingException(
-                                    parser.getTokenLocation(),
-                                    "[" + NAME + "] query does not support [" + currentFieldName + "]"
-                                );
-                            }
+                        } else {
+                            throw new ParsingException(
+                                parser.getTokenLocation(),
+                                "[" + NAME + "] query does not support [" + currentFieldName + "]"
+                            );
+                        }
                     } else {
                         throw new ParsingException(
                             parser.getTokenLocation(),
diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index c1e97e7429643..7e644a8800bbd 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -37,12 +37,10 @@
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -352,40 +350,32 @@ public static Item parse(XContentParser parser, Item item) throws IOException {
                 } else if (currentFieldName != null) {
                     if (INDEX.match(currentFieldName, parser.getDeprecationHandler())) {
                         item.index = parser.text();
-                    } else if (parser.getRestApiVersion() == RestApiVersion.V_7
-                        && TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
-                            deprecationLogger.compatibleCritical("more_like_this_query_with_types", TYPES_DEPRECATION_MESSAGE);
-                        } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
-                            item.id = parser.text();
-                        } else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) {
-                            item.doc = BytesReference.bytes(jsonBuilder().copyCurrentStructure(parser));
-                            item.xContentType = XContentType.JSON;
-                        } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
-                            if (token == XContentParser.Token.START_ARRAY) {
-                                List<String> fields = new ArrayList<>();
-                                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                                    fields.add(parser.text());
-                                }
-                                item.fields(fields.toArray(new String[fields.size()]));
-                            } else {
-                                throw new ElasticsearchParseException(
-                                    "failed to parse More Like This item. field [fields] must be an array"
-                                );
+                    } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
+                        item.id = parser.text();
+                    } else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) {
+                        item.doc = BytesReference.bytes(jsonBuilder().copyCurrentStructure(parser));
+                        item.xContentType = XContentType.JSON;
+                    } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
+                        if (token == XContentParser.Token.START_ARRAY) {
+                            List<String> fields = new ArrayList<>();
+                            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                                fields.add(parser.text());
                             }
-                        } else if (PER_FIELD_ANALYZER.match(currentFieldName, parser.getDeprecationHandler())) {
-                            item.perFieldAnalyzer(TermVectorsRequest.readPerFieldAnalyzer(parser.map()));
-                        } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) {
-                            item.routing = parser.text();
-                        } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
-                            item.version = parser.longValue();
-                        } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
-                            item.versionType = VersionType.fromString(parser.text());
+                            item.fields(fields.toArray(new String[fields.size()]));
                         } else {
-                            throw new ElasticsearchParseException(
-                                "failed to parse More Like This item. unknown field [{}]",
-                                currentFieldName
-                            );
+                            throw new ElasticsearchParseException("failed to parse More Like This item. field [fields] must be an array");
                         }
+                    } else if (PER_FIELD_ANALYZER.match(currentFieldName, parser.getDeprecationHandler())) {
+                        item.perFieldAnalyzer(TermVectorsRequest.readPerFieldAnalyzer(parser.map()));
+                    } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) {
+                        item.routing = parser.text();
+                    } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
+                        item.version = parser.longValue();
+                    } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
+                        item.versionType = VersionType.fromString(parser.text());
+                    } else {
+                        throw new ElasticsearchParseException("failed to parse More Like This item. unknown field [{}]", currentFieldName);
+                    }
                 }
             }
             if (item.id != null && item.doc != null) {
@@ -405,9 +395,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (this.index != null) {
                 builder.field(INDEX.getPreferredName(), this.index);
             }
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(TYPE.getPreferredName(), MapperService.SINGLE_MAPPING_NAME);
-            }
             if (this.id != null) {
                 builder.field(ID.getPreferredName(), this.id);
             }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
index 1deba84cce355..17e651ab24696 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.search.MatchQueryParser;
 import org.elasticsearch.index.search.MultiMatchQueryParser;
@@ -45,11 +44,7 @@
 public final class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQueryBuilder> {
 
     public static final String NAME = "multi_match";
-    private static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "cutoff_freqency is not supported."
-        + " The [multi_match] query can skip block of documents efficiently if the total number of hits is not tracked";
-    private static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency").withAllDeprecated(
-        CUTOFF_FREQUENCY_DEPRECATION_MSG
-    ).forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
+
     public static final MultiMatchQueryBuilder.Type DEFAULT_TYPE = MultiMatchQueryBuilder.Type.BEST_FIELDS;
     public static final Operator DEFAULT_OPERATOR = Operator.OR;
     public static final int DEFAULT_PHRASE_SLOP = MatchQueryParser.DEFAULT_PHRASE_SLOP;
@@ -654,15 +649,12 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws
                     autoGenerateSynonymsPhraseQuery = parser.booleanValue();
                 } else if (FUZZY_TRANSPOSITIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     fuzzyTranspositions = parser.booleanValue();
-                } else if (parser.getRestApiVersion() == RestApiVersion.V_7
-                    && CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG);
-                    } else {
-                        throw new ParsingException(
-                            parser.getTokenLocation(),
-                            "[" + NAME + "] query does not support [" + currentFieldName + "]"
-                        );
-                    }
+                } else {
+                    throw new ParsingException(
+                        parser.getTokenLocation(),
+                        "[" + NAME + "] query does not support [" + currentFieldName + "]"
+                    );
+                }
             } else {
                 throw new ParsingException(
                     parser.getTokenLocation(),
diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 55642ccf0275a..626875c75a5fe 100644
--- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -16,8 +16,8 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopDocsCollector;
-import org.apache.lucene.search.TopFieldCollector;
-import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.search.TopFieldCollectorManager;
+import org.apache.lucene.search.TopScoreDocCollectorManager;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.search.Weight;
@@ -443,12 +443,12 @@ public TopDocsAndMaxScore topDocs(SearchHit hit) throws IOException {
                 TopDocsCollector<?> topDocsCollector;
                 MaxScoreCollector maxScoreCollector = null;
                 if (sort() != null) {
-                    topDocsCollector = TopFieldCollector.create(sort().sort, topN, Integer.MAX_VALUE);
+                    topDocsCollector = new TopFieldCollectorManager(sort().sort, topN, null, Integer.MAX_VALUE, false).newCollector();
                     if (trackScores()) {
                         maxScoreCollector = new MaxScoreCollector();
                     }
                 } else {
-                    topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
+                    topDocsCollector = new TopScoreDocCollectorManager(topN, null, Integer.MAX_VALUE, false).newCollector();
                     maxScoreCollector = new MaxScoreCollector();
                 }
                 intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
diff --git a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
index 24817b778a4da..fcf986191da23 100644
--- a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -189,11 +190,24 @@ public String getWriteableName() {
     }
 
     @Override
-    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException {
+    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) {
         MappedFieldType fieldType = context.getFieldType(this.fieldName);
         if (fieldType == null) {
             return new MatchNoneQueryBuilder("The \"" + getName() + "\" query is against a field that does not exist");
-        } else if (fieldType instanceof ConstantFieldType constantFieldType) {
+        }
+        return maybeRewriteBasedOnConstantFields(fieldType, context);
+    }
+
+    @Override
+    protected QueryBuilder doCoordinatorRewrite(CoordinatorRewriteContext coordinatorRewriteContext) {
+        MappedFieldType fieldType = coordinatorRewriteContext.getFieldType(this.fieldName);
+        // we don't rewrite a null field type to `match_none` on the coordinator because the coordinator has access
+        // to only a subset of fields see {@link CoordinatorRewriteContext#getFieldType}
+        return maybeRewriteBasedOnConstantFields(fieldType, coordinatorRewriteContext);
+    }
+
+    private QueryBuilder maybeRewriteBasedOnConstantFields(@Nullable MappedFieldType fieldType, QueryRewriteContext context) {
+        if (fieldType instanceof ConstantFieldType constantFieldType) {
             // This logic is correct for all field types, but by only applying it to constant
             // fields we also have the guarantee that it doesn't perform I/O, which is important
             // since rewrites might happen on a network thread.
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
index 157ed617f3eb5..fce74aa60ab16 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
@@ -11,9 +11,12 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ResolvedIndices;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
@@ -23,6 +26,7 @@
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -235,7 +239,7 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap
             TextFieldMapper.Builder builder = new TextFieldMapper.Builder(
                 name,
                 getIndexAnalyzers(),
-                getIndexSettings() != null && getIndexSettings().getMode().isSyntheticSourceEnabled()
+                getIndexSettings() != null && SourceFieldMapper.isSynthetic(getIndexSettings())
             );
             return builder.build(MapperBuilderContext.root(false, false)).fieldType();
         } else {
@@ -406,4 +410,22 @@ public ResolvedIndices getResolvedIndices() {
     public PointInTimeBuilder getPointInTimeBuilder() {
         return pit;
     }
+
+    /**
+     * Retrieve the first tier preference from the index setting. If the setting is not
+     * present, then return null.
+     */
+    @Nullable
+    public String getTierPreference() {
+        Settings settings = getIndexSettings().getSettings();
+        String value = DataTier.TIER_PREFERENCE_SETTING.get(settings);
+
+        if (Strings.hasText(value) == false) {
+            return null;
+        }
+
+        // Tier preference can be a comma-delimited list of tiers, ordered by preference
+        // It was decided we should only test the first of these potentially multiple preferences.
+        return value.split(",")[0].trim();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/RegexpFlag.java b/server/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
index 6072a81691ffa..30921d22a8d82 100644
--- a/server/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
+++ b/server/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
@@ -10,9 +10,12 @@
 
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.UpdateForV10;
 
 import java.util.Locale;
 
+import static org.apache.lucene.util.automaton.RegExp.DEPRECATED_COMPLEMENT;
+
 /**
  * Regular expression syntax flags. Each flag represents optional syntax support in the regular expression:
  * <ul>
@@ -37,8 +40,11 @@ public enum RegexpFlag {
 
     /**
      * Enables complement expression of the form: {@code ~&lt;expression&gt;}
+     * We use the deprecated support in Lucene 10. Will be removed in Lucene 11
+     * https://github.com/elastic/elasticsearch/issues/113465
      */
-    COMPLEMENT(RegExp.COMPLEMENT),
+    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)
+    COMPLEMENT(DEPRECATED_COMPLEMENT),
 
     /**
      * Enables empty language expression: {@code #}
@@ -63,7 +69,7 @@ public enum RegexpFlag {
     /**
      * Enables all available option flags
      */
-    ALL(RegExp.ALL);
+    ALL(RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT);
 
     final int value;
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
index dc439fab58ffc..461dc66322434 100644
--- a/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
@@ -280,7 +280,9 @@ protected Query doToQuery(SearchExecutionContext context) throws QueryShardExcep
         int matchFlagsValue = caseInsensitive ? RegExp.ASCII_CASE_INSENSITIVE : 0;
         Query query = null;
         // For BWC we mask irrelevant bits (RegExp changed ALL from 0xffff to 0xff)
-        int sanitisedSyntaxFlag = syntaxFlagsValue & RegExp.ALL;
+        // We need to preserve the DEPRECATED_COMPLEMENT for now though
+        int deprecatedComplementFlag = syntaxFlagsValue & RegExp.DEPRECATED_COMPLEMENT;
+        int sanitisedSyntaxFlag = syntaxFlagsValue & (RegExp.ALL | deprecatedComplementFlag);
 
         MappedFieldType fieldType = context.getFieldType(fieldName);
         if (fieldType != null) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index c96771978bd42..8d3fd1d92e1e7 100644
--- a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -18,6 +18,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.ElasticsearchException;
@@ -184,7 +185,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
             return new ConstantScoreWeight(this, boost) {
 
                 @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
+                public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                     DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
                     final FilterScript leafScript = filterScript.newInstance(new DocValuesDocReader(lookup, context));
                     TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
@@ -201,7 +202,8 @@ public float matchCost() {
                             return 1000f;
                         }
                     };
-                    return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
+                    Scorer scorer = new ConstantScoreScorer(score(), scoreMode, twoPhase);
+                    return new DefaultScorerSupplier(scorer);
                 }
 
                 @Override
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
index 2978b3bfbf69c..113f66f3e58de 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.xcontent.ParseField;
@@ -170,11 +171,24 @@ protected void addExtraXContent(XContentBuilder builder, Params params) throws I
     }
 
     @Override
-    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException {
+    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) {
         MappedFieldType fieldType = context.getFieldType(this.fieldName);
         if (fieldType == null) {
             return new MatchNoneQueryBuilder("The \"" + getName() + "\" query is against a field that does not exist");
-        } else if (fieldType instanceof ConstantFieldType constantFieldType) {
+        }
+        return maybeRewriteBasedOnConstantFields(fieldType, context);
+    }
+
+    @Override
+    protected QueryBuilder doCoordinatorRewrite(CoordinatorRewriteContext coordinatorRewriteContext) {
+        MappedFieldType fieldType = coordinatorRewriteContext.getFieldType(this.fieldName);
+        // we don't rewrite a null field type to `match_none` on the coordinator because the coordinator has access
+        // to only a subset of fields see {@link CoordinatorRewriteContext#getFieldType}
+        return maybeRewriteBasedOnConstantFields(fieldType, coordinatorRewriteContext);
+    }
+
+    private QueryBuilder maybeRewriteBasedOnConstantFields(@Nullable MappedFieldType fieldType, QueryRewriteContext context) {
+        if (fieldType instanceof ConstantFieldType constantFieldType) {
             // This logic is correct for all field types, but by only applying it to constant
             // fields we also have the guarantee that it doesn't perform I/O, which is important
             // since rewrites might happen on a network thread.
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index 4035bc02fba79..dec4090a3e6bd 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -393,11 +393,24 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
     }
 
     @Override
-    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException {
+    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) {
         MappedFieldType fieldType = context.getFieldType(this.fieldName);
         if (fieldType == null) {
             return new MatchNoneQueryBuilder("The \"" + getName() + "\" query is against a field that does not exist");
-        } else if (fieldType instanceof ConstantFieldType constantFieldType) {
+        }
+        return maybeRewriteBasedOnConstantFields(fieldType, context);
+    }
+
+    @Override
+    protected QueryBuilder doCoordinatorRewrite(CoordinatorRewriteContext coordinatorRewriteContext) {
+        MappedFieldType fieldType = coordinatorRewriteContext.getFieldType(this.fieldName);
+        // we don't rewrite a null field type to `match_none` on the coordinator because the coordinator has access
+        // to only a subset of fields see {@link CoordinatorRewriteContext#getFieldType}
+        return maybeRewriteBasedOnConstantFields(fieldType, coordinatorRewriteContext);
+    }
+
+    private QueryBuilder maybeRewriteBasedOnConstantFields(@Nullable MappedFieldType fieldType, QueryRewriteContext context) {
+        if (fieldType instanceof ConstantFieldType constantFieldType) {
             // This logic is correct for all field types, but by only applying it to constant
             // fields we also have the guarantee that it doesn't perform I/O, which is important
             // since rewrites might happen on a network thread.
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
index 81afdf0ebe5e0..a6116ccf2c495 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.sandbox.search.CoveringQuery;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DoubleValues;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LongValues;
@@ -273,8 +272,8 @@ protected Query doToQuery(SearchExecutionContext context) {
             return Queries.newMatchNoDocsQuery("No terms supplied for \"" + getName() + "\" query.");
         }
         // Fail before we attempt to create the term queries:
-        if (values.size() > BooleanQuery.getMaxClauseCount()) {
-            throw new BooleanQuery.TooManyClauses();
+        if (values.size() > IndexSearcher.getMaxClauseCount()) {
+            throw new IndexSearcher.TooManyClauses();
         }
 
         List<Query> queries = createTermQueries(context);
diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java
index f707684cb46e5..c9aae0195acf7 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -26,6 +27,7 @@
 
 import java.io.IOException;
 
+@UpdateForV9(owner = UpdateForV9.Owner.SEARCH_RELEVANCE) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
 public class TypeQueryV7Builder extends AbstractQueryBuilder<TypeQueryV7Builder> {
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TypeQueryV7Builder.class);
     public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Type queries are deprecated, "
diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
index f287812ebbc10..419195e5e5ba5 100644
--- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.support.QueryParsers;
@@ -200,11 +201,24 @@ public static WildcardQueryBuilder fromXContent(XContentParser parser) throws IO
     }
 
     @Override
-    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException {
+    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) {
         MappedFieldType fieldType = context.getFieldType(this.fieldName);
         if (fieldType == null) {
-            return new MatchNoneQueryBuilder("The \"" + getName() + "\" query  is against a field that does not exist");
-        } else if (fieldType instanceof ConstantFieldType constantFieldType) {
+            return new MatchNoneQueryBuilder("The \"" + getName() + "\" query is against a field that does not exist");
+        }
+        return maybeRewriteBasedOnConstantFields(fieldType, context);
+    }
+
+    @Override
+    protected QueryBuilder doCoordinatorRewrite(CoordinatorRewriteContext coordinatorRewriteContext) {
+        MappedFieldType fieldType = coordinatorRewriteContext.getFieldType(this.fieldName);
+        // we don't rewrite a null field type to `match_none` on the coordinator because the coordinator has access
+        // to only a subset of fields see {@link CoordinatorRewriteContext#getFieldType}
+        return maybeRewriteBasedOnConstantFields(fieldType, coordinatorRewriteContext);
+    }
+
+    private QueryBuilder maybeRewriteBasedOnConstantFields(@Nullable MappedFieldType fieldType, QueryRewriteContext context) {
+        if (fieldType instanceof ConstantFieldType constantFieldType) {
             // This logic is correct for all field types, but by only applying it to constant
             // fields we also have the guarantee that it doesn't perform I/O, which is important
             // since rewrites might happen on a network thread.
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
index 528f0bd6dae08..1327721a88427 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
@@ -149,7 +149,7 @@ private static Response wrapSearchResponse(SearchResponse response) {
             }
             hits = unmodifiableList(hits);
         }
-        long total = response.getHits().getTotalHits().value;
+        long total = response.getHits().getTotalHits().value();
         return new Response(response.isTimedOut(), failures, total, hits, response.getScrollId());
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index b7ec486df612a..f9d027e0c9c1c 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -20,8 +20,8 @@
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -354,20 +354,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             ObjectParser.ValueType.OBJECT
         );
 
-        PARSER.declareInt(
-            ReindexRequest::setMaxDocsValidateIdentical,
-            new ParseField("max_docs", "size").forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))
-        );
+        PARSER.declareInt(ReindexRequest::setMaxDocsValidateIdentical, new ParseField("max_docs"));
 
-        PARSER.declareInt(
-            ReindexRequest::setMaxDocsValidateIdentical,
-            new ParseField("max_docs").forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8))
-        );
         // avoid silently accepting an ignored size.
-        PARSER.declareInt(
-            (r, s) -> failOnSizeSpecified(),
-            new ParseField("size").forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8))
-        );
+        PARSER.declareInt((r, s) -> failOnSizeSpecified(), new ParseField("size"));
 
         PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p)), new ParseField("script"), ObjectParser.ValueType.OBJECT);
         PARSER.declareString(ReindexRequest::setConflicts, new ParseField("conflicts"));
@@ -509,6 +499,8 @@ static void setMaxDocsValidateIdentical(AbstractBulkByScrollRequest<?> request,
         }
     }
 
+    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_INDEXING)
+    // do we still need this ref to [max_docs] or can we remove the field entirely so it's rejected with the default message?
     private static void failOnSizeSpecified() {
         throw new IllegalArgumentException("invalid parameter [size], use [max_docs] instead");
     }
diff --git a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
index 505c20f642093..5f135c674ba1a 100644
--- a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
@@ -26,6 +26,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostAttribute;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.QueryBuilder;
@@ -690,7 +691,7 @@ private Query analyzeGraphPhrase(TokenStream source, String field, Type type, in
             List<SpanQuery> clauses = new ArrayList<>();
             int[] articulationPoints = graph.articulationPoints();
             int lastState = 0;
-            int maxClauseCount = BooleanQuery.getMaxClauseCount();
+            int maxClauseCount = IndexSearcher.getMaxClauseCount();
             for (int i = 0; i <= articulationPoints.length; i++) {
                 int start = lastState;
                 int end = -1;
@@ -708,7 +709,7 @@ private Query analyzeGraphPhrase(TokenStream source, String field, Type type, in
                         SpanQuery q = createSpanQuery(ts, field, usePrefix);
                         if (q != null) {
                             if (queries.size() >= maxClauseCount) {
-                                throw new BooleanQuery.TooManyClauses();
+                                throw new IndexSearcher.TooManyClauses();
                             }
                             queries.add(q);
                         }
@@ -722,14 +723,14 @@ private Query analyzeGraphPhrase(TokenStream source, String field, Type type, in
                     Term[] terms = graph.getTerms(field, start);
                     assert terms.length > 0;
                     if (terms.length >= maxClauseCount) {
-                        throw new BooleanQuery.TooManyClauses();
+                        throw new IndexSearcher.TooManyClauses();
                     }
                     queryPos = newSpanQuery(terms, usePrefix);
                 }
 
                 if (queryPos != null) {
                     if (clauses.size() >= maxClauseCount) {
-                        throw new BooleanQuery.TooManyClauses();
+                        throw new IndexSearcher.TooManyClauses();
                     }
                     clauses.add(queryPos);
                 }
diff --git a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java
index 52122ed86ef69..446d78078e642 100644
--- a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java
@@ -200,7 +200,7 @@ protected Query createBooleanPrefixQuery(String field, String queryText, Boolean
         protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
             BytesRef[] values = new BytesRef[terms.length];
             for (int i = 0; i < terms.length; i++) {
-                values[i] = terms[i].term;
+                values[i] = terms[i].term();
             }
             return blendTerms(context, values, tieBreaker, lenient, blendedFields);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java b/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java
index db0077284bbd3..96e8ac35c8e32 100644
--- a/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java
+++ b/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java
@@ -24,8 +24,6 @@
 import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.NestedObjectMapper;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.function.Predicate;
 
 /** Utility class to filter parent and children clauses when building nested
@@ -55,15 +53,10 @@ public boolean mightMatchNestedDocs(Query query) {
             // cover a high majority of use-cases
             return mightMatchNestedDocs(((TermQuery) query).getTerm().field());
         } else if (query instanceof TermInSetQuery tis) {
-            try {
-                if (tis.getTermsCount() > 0) {
-                    return mightMatchNestedDocs(tis.getField());
-                } else {
-                    return false;
-                }
-            } catch (IOException e) {
-                // this handling isn't needed any more once we move to Lucene 10
-                throw new UncheckedIOException("We are not doing IO here, this should never happen.", e);
+            if (tis.getTermsCount() > 0) {
+                return mightMatchNestedDocs(tis.getField());
+            } else {
+                return false;
             }
         } else if (query instanceof PointRangeQuery) {
             return mightMatchNestedDocs(((PointRangeQuery) query).getField());
@@ -75,13 +68,13 @@ public boolean mightMatchNestedDocs(Query query) {
                 return bq.clauses()
                     .stream()
                     .filter(BooleanClause::isRequired)
-                    .map(BooleanClause::getQuery)
+                    .map(BooleanClause::query)
                     .allMatch(this::mightMatchNestedDocs);
             } else {
                 return bq.clauses()
                     .stream()
-                    .filter(c -> c.getOccur() == Occur.SHOULD)
-                    .map(BooleanClause::getQuery)
+                    .filter(c -> c.occur() == Occur.SHOULD)
+                    .map(BooleanClause::query)
                     .anyMatch(this::mightMatchNestedDocs);
             }
         } else if (query instanceof ESToParentBlockJoinQuery) {
@@ -122,15 +115,10 @@ public boolean mightMatchNonNestedDocs(Query query, String nestedPath) {
         } else if (query instanceof TermQuery) {
             return mightMatchNonNestedDocs(((TermQuery) query).getTerm().field(), nestedPath);
         } else if (query instanceof TermInSetQuery tis) {
-            try {
-                if (tis.getTermsCount() > 0) {
-                    return mightMatchNonNestedDocs(tis.getField(), nestedPath);
-                } else {
-                    return false;
-                }
-            } catch (IOException e) {
-                // this handling isn't needed any more once we move to Lucene 10
-                throw new UncheckedIOException("We are not doing IO here, this should never happen.", e);
+            if (tis.getTermsCount() > 0) {
+                return mightMatchNonNestedDocs(tis.getField(), nestedPath);
+            } else {
+                return false;
             }
         } else if (query instanceof PointRangeQuery) {
             return mightMatchNonNestedDocs(((PointRangeQuery) query).getField(), nestedPath);
@@ -142,13 +130,13 @@ public boolean mightMatchNonNestedDocs(Query query, String nestedPath) {
                 return bq.clauses()
                     .stream()
                     .filter(BooleanClause::isRequired)
-                    .map(BooleanClause::getQuery)
+                    .map(BooleanClause::query)
                     .allMatch(q -> mightMatchNonNestedDocs(q, nestedPath));
             } else {
                 return bq.clauses()
                     .stream()
-                    .filter(c -> c.getOccur() == Occur.SHOULD)
-                    .map(BooleanClause::getQuery)
+                    .filter(c -> c.occur() == Occur.SHOULD)
+                    .map(BooleanClause::query)
                     .anyMatch(q -> mightMatchNonNestedDocs(q, nestedPath));
             }
         } else {
@@ -183,5 +171,4 @@ boolean mightMatchNonNestedDocs(String field, String nestedPath) {
         }
         return true;
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
index 76dba60689422..d237a03335337 100644
--- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
@@ -760,7 +760,14 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc
                 setAnalyzer(forceAnalyzer);
                 return super.getRegexpQuery(field, termStr);
             }
-            return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getDeterminizeWorkLimit(), getMultiTermRewriteMethod(), context);
+            return currentFieldType.regexpQuery(
+                termStr,
+                RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT,
+                0,
+                getDeterminizeWorkLimit(),
+                getMultiTermRewriteMethod(),
+                context
+            );
         } catch (RuntimeException e) {
             if (lenient) {
                 return newLenientFieldQuery(field, e);
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
index e67e878fd3827..f1e3ac270d959 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.gateway.WriteStateException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersions;
@@ -471,9 +470,9 @@ public RetentionLeases loadRetentionLeases(final Path path) throws IOException {
         return emptyIfNull(retentionLeases);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_INDEXING)
     private static RetentionLeases emptyIfNull(RetentionLeases retentionLeases) {
-        // we expect never to see a null in 8.x, so adjust this to throw an exception from v9 onwards.
+        // `MetadataStateFormat.FORMAT.loadLatestState` can actually return null when the state directory
+        // on a node hasn't been initialized yet
         return retentionLeases == null ? RetentionLeases.EMPTY : retentionLeases;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
index 81b6ad136f129..cdeaa177dfd5e 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
@@ -95,14 +94,9 @@ protected void doExecute(Task task, Request request, ActionListener<ReplicationR
     }
 
     final void backgroundSync(ShardId shardId, String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases) {
-        final ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+        try (var ignore = threadPool.getThreadContext().newEmptySystemContext()) {
             // we have to execute under the system context so that if security is enabled the sync is authorized
-            threadContext.markAsSystemContext();
-            final Request request = new Request(shardId, retentionLeases);
-            try (var ignored = threadContext.newTraceContext()) {
-                sendRetentionLeaseSyncAction(shardId, primaryAllocationId, primaryTerm, request);
-            }
+            sendRetentionLeaseSyncAction(shardId, primaryAllocationId, primaryTerm, new Request(shardId, retentionLeases));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
index 1f683503c23c8..4a2b54b0c7b80 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.index.shard.IndexShard;
@@ -111,47 +110,43 @@ final void sync(
         RetentionLeases retentionLeases,
         ActionListener<ReplicationResponse> listener
     ) {
-        final ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+        try (var ignore = threadPool.getThreadContext().newEmptySystemContext()) {
             // we have to execute under the system context so that if security is enabled the sync is authorized
-            threadContext.markAsSystemContext();
             final Request request = new Request(shardId, retentionLeases);
-            try (var ignored = threadContext.newTraceContext()) {
-                final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_sync", request);
-                transportService.sendChildRequest(
-                    clusterService.localNode(),
-                    transportPrimaryAction,
-                    new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm),
-                    task,
-                    transportOptions,
-                    new TransportResponseHandler<ReplicationResponse>() {
-                        @Override
-                        public ReplicationResponse read(StreamInput in) throws IOException {
-                            return newResponseInstance(in);
-                        }
-
-                        @Override
-                        public Executor executor() {
-                            return TransportResponseHandler.TRANSPORT_WORKER;
-                        }
-
-                        @Override
-                        public void handleResponse(ReplicationResponse response) {
-                            task.setPhase("finished");
-                            taskManager.unregister(task);
-                            listener.onResponse(response);
-                        }
-
-                        @Override
-                        public void handleException(TransportException e) {
-                            LOGGER.log(getExceptionLogLevel(e), () -> format("%s retention lease sync failed", shardId), e);
-                            task.setPhase("finished");
-                            taskManager.unregister(task);
-                            listener.onFailure(e);
-                        }
+            final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_sync", request);
+            transportService.sendChildRequest(
+                clusterService.localNode(),
+                transportPrimaryAction,
+                new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm),
+                task,
+                transportOptions,
+                new TransportResponseHandler<ReplicationResponse>() {
+                    @Override
+                    public ReplicationResponse read(StreamInput in) throws IOException {
+                        return newResponseInstance(in);
                     }
-                );
-            }
+
+                    @Override
+                    public Executor executor() {
+                        return TransportResponseHandler.TRANSPORT_WORKER;
+                    }
+
+                    @Override
+                    public void handleResponse(ReplicationResponse response) {
+                        task.setPhase("finished");
+                        taskManager.unregister(task);
+                        listener.onResponse(response);
+                    }
+
+                    @Override
+                    public void handleException(TransportException e) {
+                        LOGGER.log(getExceptionLogLevel(e), () -> format("%s retention lease sync failed", shardId), e);
+                        task.setPhase("finished");
+                        taskManager.unregister(task);
+                        listener.onFailure(e);
+                    }
+                }
+            );
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
index b4a8610ded04d..b0a4d333ba77f 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ToXContent;
@@ -305,13 +304,6 @@ public Stats getTotal() {
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject(Fields.INDEXING);
         totalStats.toXContent(builder, params);
-        if (builder.getRestApiVersion() == RestApiVersion.V_7 && params.param("types") != null) {
-            builder.startObject(Fields.TYPES);
-            builder.startObject(MapperService.SINGLE_MAPPING_NAME);
-            totalStats.toXContent(builder, params);
-            builder.endObject();
-            builder.endObject();
-        }
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
index 8a54b842a3792..8a2fc396c6737 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -147,8 +147,7 @@ public boolean addOrNotify(Translog.Location location, Consumer<Boolean> listene
             if (refreshForcers == 0 && roomForListener(maxRefreshes, listeners, checkpointRefreshListeners)) {
                 ThreadContext.StoredContext storedContext = threadContext.newStoredContextPreservingResponseHeaders();
                 Consumer<Boolean> contextPreservingListener = forced -> {
-                    try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-                        storedContext.restore();
+                    try (var ignore = threadContext.restoreExistingContext(storedContext)) {
                         listener.accept(forced);
                     }
                 };
diff --git a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedLuceneSegmentsAction.java b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedLuceneSegmentsAction.java
index 562bf1e75dc1f..97d1b3342ca2b 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedLuceneSegmentsAction.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedLuceneSegmentsAction.java
@@ -33,7 +33,7 @@ public static Tuple<RemoveCorruptedShardDataCommand.CleanStatus, String> getClea
 
         final CheckIndex.Status status;
         try (CheckIndex checker = new CheckIndex(indexDirectory, writeLock)) {
-            checker.setChecksumsOnly(true);
+            checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_CHECKSUM_CHECKS);
             checker.setInfoStream(printStream, verbose);
 
             status = checker.checkIndex(null);
@@ -64,7 +64,7 @@ public static void execute(Terminal terminal, Directory indexDirectory, Lock wri
         final CheckIndex.Status status;
         try (CheckIndex checker = new CheckIndex(indexDirectory, writeLock)) {
 
-            checker.setChecksumsOnly(true);
+            checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_CHECKSUM_CHECKS);
             checker.setInfoStream(printStream, verbose);
 
             status = checker.checkIndex(null);
diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java
index f1291ac6faa51..94a29258f3202 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.join.BitSetProducer;
@@ -73,7 +74,7 @@ public String toString() {
             }
 
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 LeafReader leafReader = context.reader();
                 FixedBitSet bitSet = new FixedBitSet(leafReader.maxDoc());
                 Terms terms = leafReader.terms(RoutingFieldMapper.NAME);
@@ -82,87 +83,102 @@ public Scorer scorer(LeafReaderContext context) throws IOException {
                     int targetShardId = indexRouting.getShard(Uid.decodeId(ref.bytes, ref.offset, ref.length), null);
                     return shardId == targetShardId;
                 };
-                if (terms == null) {
-                    // this is the common case - no partitioning and no _routing values
-                    // in this case we also don't do anything special with regards to nested docs since we basically delete
-                    // by ID and parent and nested all have the same id.
-                    assert indexMetadata.isRoutingPartitionedIndex() == false;
-                    findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, bitSet::set);
-                } else {
-                    final BitSet parentBitSet;
-                    if (nestedParentBitSetProducer == null) {
-                        parentBitSet = null;
-                    } else {
-                        parentBitSet = nestedParentBitSetProducer.getBitSet(context);
-                        if (parentBitSet == null) {
-                            return null; // no matches
-                        }
-                    }
-                    if (indexMetadata.isRoutingPartitionedIndex()) {
-                        // this is the heaviest invariant. Here we have to visit all docs stored fields do extract _id and _routing
-                        // this index is routing partitioned.
-                        Visitor visitor = new Visitor(leafReader);
-                        TwoPhaseIterator twoPhaseIterator = parentBitSet == null
-                            ? new RoutingPartitionedDocIdSetIterator(visitor)
-                            : new NestedRoutingPartitionedDocIdSetIterator(visitor, parentBitSet);
-                        return new ConstantScoreScorer(this, score(), scoreMode, twoPhaseIterator);
-                    } else {
-                        // here we potentially guard the docID consumers with our parent bitset if we have one.
-                        // this ensures that we are only marking root documents in the nested case and if necessary
-                        // we do a second pass to mark the corresponding children in markChildDocs
-                        Function<IntConsumer, IntConsumer> maybeWrapConsumer = consumer -> {
-                            if (parentBitSet != null) {
-                                return docId -> {
-                                    if (parentBitSet.get(docId)) {
-                                        consumer.accept(docId);
+
+                return new ScorerSupplier() {
+                    @Override
+                    public Scorer get(long leadCost) throws IOException {
+                        if (terms == null) {
+                            // this is the common case - no partitioning and no _routing values
+                            // in this case we also don't do anything special with regards to nested docs since we basically delete
+                            // by ID and parent and nested all have the same id.
+                            assert indexMetadata.isRoutingPartitionedIndex() == false;
+                            findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, bitSet::set);
+                        } else {
+                            final BitSet parentBitSet;
+                            if (nestedParentBitSetProducer == null) {
+                                parentBitSet = null;
+                            } else {
+                                parentBitSet = nestedParentBitSetProducer.getBitSet(context);
+                                if (parentBitSet == null) {
+                                    return null; // no matches
+                                }
+                            }
+                            if (indexMetadata.isRoutingPartitionedIndex()) {
+                                // this is the heaviest invariant. Here we have to visit all docs stored fields do extract _id and _routing
+                                // this index is routing partitioned.
+                                Visitor visitor = new Visitor(leafReader);
+                                TwoPhaseIterator twoPhaseIterator = parentBitSet == null
+                                    ? new RoutingPartitionedDocIdSetIterator(visitor)
+                                    : new NestedRoutingPartitionedDocIdSetIterator(visitor, parentBitSet);
+                                return new ConstantScoreScorer(score(), scoreMode, twoPhaseIterator);
+                            } else {
+                                // here we potentially guard the docID consumers with our parent bitset if we have one.
+                                // this ensures that we are only marking root documents in the nested case and if necessary
+                                // we do a second pass to mark the corresponding children in markChildDocs
+                                Function<IntConsumer, IntConsumer> maybeWrapConsumer = consumer -> {
+                                    if (parentBitSet != null) {
+                                        return docId -> {
+                                            if (parentBitSet.get(docId)) {
+                                                consumer.accept(docId);
+                                            }
+                                        };
                                     }
+                                    return consumer;
                                 };
-                            }
-                            return consumer;
-                        };
-                        // in the _routing case we first go and find all docs that have a routing value and mark the ones we have to delete
-                        findSplitDocs(RoutingFieldMapper.NAME, ref -> {
-                            int targetShardId = indexRouting.getShard(null, ref.utf8ToString());
-                            return shardId == targetShardId;
-                        }, leafReader, maybeWrapConsumer.apply(bitSet::set));
-
-                        // TODO have the IndexRouting build the query and pass routingRequired in
-                        boolean routingRequired = indexMetadata.mapping() == null ? false : indexMetadata.mapping().routingRequired();
-                        // now if we have a mixed index where some docs have a _routing value and some don't we have to exclude the ones
-                        // with a routing value from the next iteration and delete / select based on the ID.
-                        if (routingRequired == false && terms.getDocCount() != leafReader.maxDoc()) {
-                            /*
-                             * This is a special case where some docs don't have routing values.
-                             * It's annoying, but it's allowed to build an index where some documents
-                             * hve routing and others don't.
-                             *
-                             * Luckily, if the routing field is required in the mapping then we can
-                             * safely assume that all documents which are don't have a routing are
-                             * nested documents. And we pick those up later based on the assignment
-                             * of the document that contains them.
-                             */
-                            FixedBitSet hasRoutingValue = new FixedBitSet(leafReader.maxDoc());
-                            findSplitDocs(
-                                RoutingFieldMapper.NAME,
-                                Predicates.never(),
-                                leafReader,
-                                maybeWrapConsumer.apply(hasRoutingValue::set)
-                            );
-                            IntConsumer bitSetConsumer = maybeWrapConsumer.apply(bitSet::set);
-                            findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, docId -> {
-                                if (hasRoutingValue.get(docId) == false) {
-                                    bitSetConsumer.accept(docId);
+                                // in the _routing case we first go and find all docs that have a routing value and mark the ones we have to
+                                // delete
+                                findSplitDocs(RoutingFieldMapper.NAME, ref -> {
+                                    int targetShardId = indexRouting.getShard(null, ref.utf8ToString());
+                                    return shardId == targetShardId;
+                                }, leafReader, maybeWrapConsumer.apply(bitSet::set));
+
+                                // TODO have the IndexRouting build the query and pass routingRequired in
+                                boolean routingRequired = indexMetadata.mapping() == null
+                                    ? false
+                                    : indexMetadata.mapping().routingRequired();
+                                // now if we have a mixed index where some docs have a _routing value and some don't we have to exclude the
+                                // ones
+                                // with a routing value from the next iteration and delete / select based on the ID.
+                                if (routingRequired == false && terms.getDocCount() != leafReader.maxDoc()) {
+                                    /*
+                                     * This is a special case where some docs don't have routing values.
+                                     * It's annoying, but it's allowed to build an index where some documents
+                                     * hve routing and others don't.
+                                     *
+                                     * Luckily, if the routing field is required in the mapping then we can
+                                     * safely assume that all documents which are don't have a routing are
+                                     * nested documents. And we pick those up later based on the assignment
+                                     * of the document that contains them.
+                                     */
+                                    FixedBitSet hasRoutingValue = new FixedBitSet(leafReader.maxDoc());
+                                    findSplitDocs(
+                                        RoutingFieldMapper.NAME,
+                                        Predicates.never(),
+                                        leafReader,
+                                        maybeWrapConsumer.apply(hasRoutingValue::set)
+                                    );
+                                    IntConsumer bitSetConsumer = maybeWrapConsumer.apply(bitSet::set);
+                                    findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, docId -> {
+                                        if (hasRoutingValue.get(docId) == false) {
+                                            bitSetConsumer.accept(docId);
+                                        }
+                                    });
                                 }
-                            });
+                            }
+                            if (parentBitSet != null) {
+                                // if nested docs are involved we also need to mark all child docs that belong to a matching parent doc.
+                                markChildDocs(parentBitSet, bitSet);
+                            }
                         }
+
+                        return new ConstantScoreScorer(score(), scoreMode, new BitSetIterator(bitSet, bitSet.length()));
                     }
-                    if (parentBitSet != null) {
-                        // if nested docs are involved we also need to mark all child docs that belong to a matching parent doc.
-                        markChildDocs(parentBitSet, bitSet);
-                    }
-                }
 
-                return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bitSet, bitSet.length()));
+                    @Override
+                    public long cost() {
+                        return leafReader.maxDoc();
+                    }
+                };
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
index c755862d9e828..b5197274dd519 100644
--- a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
+++ b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
@@ -80,7 +80,7 @@ public IndexingPressureStats(StreamInput in) throws IOException {
             primaryDocumentRejections = -1L;
         }
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.INDEXING_PRESSURE_REQUEST_REJECTIONS_COUNT)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             totalCoordinatingRequests = in.readVLong();
         } else {
             totalCoordinatingRequests = -1L;
@@ -157,7 +157,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(primaryDocumentRejections);
         }
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.INDEXING_PRESSURE_REQUEST_REJECTIONS_COUNT)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeVLong(totalCoordinatingRequests);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java
index 3dc5953e3d3d8..bc94db13074db 100644
--- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java
+++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java
@@ -33,6 +33,7 @@
 import java.nio.file.Path;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.function.BiPredicate;
 
 public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory {
 
@@ -67,12 +68,12 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
                 // Use Lucene defaults
                 final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
                 if (primaryDirectory instanceof MMapDirectory mMapDirectory) {
-                    return new HybridDirectory(lockFactory, setPreload(mMapDirectory, lockFactory, preLoadExtensions));
+                    return new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
                 } else {
                     return primaryDirectory;
                 }
             case MMAPFS:
-                return setPreload(new MMapDirectory(location, lockFactory), lockFactory, preLoadExtensions);
+                return setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
             case SIMPLEFS:
             case NIOFS:
                 return new NIOFSDirectory(location, lockFactory);
@@ -81,17 +82,23 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
         }
     }
 
-    public static MMapDirectory setPreload(MMapDirectory mMapDirectory, LockFactory lockFactory, Set<String> preLoadExtensions)
-        throws IOException {
-        assert mMapDirectory.getPreload() == false;
+    /** Sets the preload, if any, on the given directory based on the extensions. Returns the same directory instance. */
+    // visibility and extensibility for testing
+    public MMapDirectory setPreload(MMapDirectory mMapDirectory, Set<String> preLoadExtensions) {
+        mMapDirectory.setPreload(getPreloadFunc(preLoadExtensions));
+        return mMapDirectory;
+    }
+
+    /** Gets a preload function based on the given preLoadExtensions. */
+    static BiPredicate<String, IOContext> getPreloadFunc(Set<String> preLoadExtensions) {
         if (preLoadExtensions.isEmpty() == false) {
             if (preLoadExtensions.contains("*")) {
-                mMapDirectory.setPreload(true);
+                return MMapDirectory.ALL_FILES;
             } else {
-                return new PreLoadMMapDirectory(mMapDirectory, lockFactory, preLoadExtensions);
+                return (name, context) -> preLoadExtensions.contains(FileSwitchDirectory.getExtension(name));
             }
         }
-        return mMapDirectory;
+        return MMapDirectory.NO_FILES;
     }
 
     /**
@@ -116,6 +123,8 @@ public IndexInput openInput(String name, IOContext context) throws IOException {
                 // we need to do these checks on the outer directory since the inner doesn't know about pending deletes
                 ensureOpen();
                 ensureCanRead(name);
+                // we switch the context here since mmap checks for the READONCE context by identity
+                context = context == Store.READONCE_CHECKSUM ? IOContext.READONCE : context;
                 // we only use the mmap to open inputs. Everything else is managed by the NIOFSDirectory otherwise
                 // we might run into trouble with files that are pendingDelete in one directory but still
                 // listed in listAll() from the other. We on the other hand don't want to list files from both dirs
@@ -162,50 +171,4 @@ MMapDirectory getDelegate() {
             return delegate;
         }
     }
-
-    // TODO it would be nice to share code between PreLoadMMapDirectory and HybridDirectory but due to the nesting aspect of
-    // directories here makes it tricky. It would be nice to allow MMAPDirectory to pre-load on a per IndexInput basis.
-    static final class PreLoadMMapDirectory extends MMapDirectory {
-        private final MMapDirectory delegate;
-        private final Set<String> preloadExtensions;
-
-        PreLoadMMapDirectory(MMapDirectory delegate, LockFactory lockFactory, Set<String> preload) throws IOException {
-            super(delegate.getDirectory(), lockFactory);
-            super.setPreload(false);
-            this.delegate = delegate;
-            this.delegate.setPreload(true);
-            this.preloadExtensions = preload;
-            assert getPreload() == false;
-        }
-
-        @Override
-        public void setPreload(boolean preload) {
-            throw new IllegalArgumentException("can't set preload on a preload-wrapper");
-        }
-
-        @Override
-        public IndexInput openInput(String name, IOContext context) throws IOException {
-            if (useDelegate(name)) {
-                // we need to do these checks on the outer directory since the inner doesn't know about pending deletes
-                ensureOpen();
-                ensureCanRead(name);
-                return delegate.openInput(name, context);
-            }
-            return super.openInput(name, context);
-        }
-
-        @Override
-        public synchronized void close() throws IOException {
-            IOUtils.close(super::close, delegate);
-        }
-
-        boolean useDelegate(String name) {
-            final String extension = FileSwitchDirectory.getExtension(name);
-            return preloadExtensions.contains(extension);
-        }
-
-        MMapDirectory getDelegate() {
-            return delegate;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java
index 186aff230b8d0..387385ea2d6a4 100644
--- a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java
+++ b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java
@@ -81,7 +81,9 @@ public enum LuceneFilesExtensions {
     VEM("vem", "Vector Metadata", true, false),
     VEMF("vemf", "Flat Vector Metadata", true, false),
     VEMQ("vemq", "Scalar Quantized Vector Metadata", true, false),
-    VEQ("veq", "Scalar Quantized Vector Data", false, true);
+    VEQ("veq", "Scalar Quantized Vector Data", false, true),
+    VEMB("vemb", "Binarized Vector Metadata", true, false),
+    VEB("veb", "Binarized Vector Data", false, true);
 
     /**
      * Allow plugin developers of custom codecs to opt out of the assertion in {@link #fromExtension}
diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java
index a1038356735f0..c3d21b23d6a49 100644
--- a/server/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/server/src/main/java/org/elasticsearch/index/store/Store.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.NIOFSDirectory;
+import org.apache.lucene.store.ReadAdvice;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
@@ -147,7 +148,15 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
      * Specific {@link IOContext} indicating that we will read only the Lucene file footer (containing the file checksum)
      * See {@link MetadataSnapshot#checksumFromLuceneFile}.
      */
-    public static final IOContext READONCE_CHECKSUM = new IOContext(IOContext.READONCE, true);
+    public static final IOContext READONCE_CHECKSUM = createReadOnceContext();
+
+    // while equivalent, these different read once contexts are checked by identity in directory implementations
+    private static IOContext createReadOnceContext() {
+        var context = IOContext.READONCE.withReadAdvice(ReadAdvice.SEQUENTIAL);
+        assert context != IOContext.READONCE;
+        assert context.equals(IOContext.READONCE);
+        return context;
+    }
 
     private final AtomicBoolean isClosed = new AtomicBoolean(false);
     private final StoreDirectory directory;
@@ -632,7 +641,7 @@ private static void failIfCorrupted(Directory directory) throws IOException {
         List<CorruptIndexException> ex = new ArrayList<>();
         for (String file : files) {
             if (file.startsWith(CORRUPTED_MARKER_NAME_PREFIX)) {
-                try (ChecksumIndexInput input = directory.openChecksumInput(file, IOContext.READONCE)) {
+                try (ChecksumIndexInput input = directory.openChecksumInput(file)) {
                     CodecUtil.checkHeader(input, CODEC, CORRUPTED_MARKER_CODEC_VERSION, CORRUPTED_MARKER_CODEC_VERSION);
                     final int size = input.readVInt();
                     final byte[] buffer = new byte[size];
@@ -919,7 +928,10 @@ private static void checksumFromLuceneFile(
             boolean readFileAsHash,
             BytesRef writerUuid
         ) throws IOException {
-            try (IndexInput in = directory.openInput(file, READONCE_CHECKSUM)) {
+            // We select the read once context carefully here since these constants, while equivalent are
+            // checked by identity in the different directory implementations.
+            var context = file.startsWith(IndexFileNames.SEGMENTS) ? IOContext.READONCE : READONCE_CHECKSUM;
+            try (IndexInput in = directory.openInput(file, context)) {
                 final long length = in.length();
                 if (length < CodecUtil.footerLength()) {
                     // If the file isn't long enough to contain the footer then verifying it triggers an IAE, but really it's corrupted
diff --git a/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java b/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java
index 2be9d0f224e24..501c2496aacb6 100644
--- a/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java
+++ b/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java
@@ -195,7 +195,7 @@ public BytesRef hash() {
      *
      * This ID may be {@link StoreFileMetadata#UNAVAILABLE_WRITER_UUID} (i.e. zero-length) if unavailable, e.g.:
      *
-     * - The file was written by a version of Lucene prior to {@link org.apache.lucene.util.Version#LUCENE_8_6_0}.
+     * - The file was written by a version of Lucene prior to 8.6.0.
      * - The metadata came from a version of Elasticsearch prior to {@link StoreFileMetadata#WRITER_UUID_MIN_VERSION}).
      * - The file is not one of the files listed above.
      *
diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 763abb41797b5..db84be817bbd7 100644
--- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -96,7 +96,7 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ
             /* or from an existing document */
             else if (docIdAndVersion != null) {
                 // fields with stored term vectors
-                termVectorsByField = docIdAndVersion.reader.getTermVectors(docIdAndVersion.docId);
+                termVectorsByField = docIdAndVersion.reader.termVectors().get(docIdAndVersion.docId);
                 Set<String> selectedFields = request.selectedFields();
                 // generate tvs for fields where analyzer is overridden
                 if (selectedFields == null && request.perFieldAnalyzer() != null) {
@@ -301,7 +301,7 @@ private static Fields generateTermVectors(
             }
         }
         /* and read vectors from it */
-        return index.createSearcher().getIndexReader().getTermVectors(0);
+        return index.createSearcher().getIndexReader().termVectors().get(0);
     }
 
     private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java
index 29161814e7724..525da1670f900 100644
--- a/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java
+++ b/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java
@@ -94,7 +94,7 @@ static Automaton buildAutomaton(String pattern) {
         String output = pattern;
         output = output.replace(".", "\\.");
         output = output.replace("*", ".*");
-        return new RegExp(output).toAutomaton();
+        return new RegExp(output, RegExp.ALL | RegExp.ALL).toAutomaton();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
index abba6ec6ae684..9bca59e9e4d62 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
@@ -12,13 +12,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.LRUQueryCache;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
-import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.common.lucene.ShardCoreKeyMap;
@@ -173,24 +171,12 @@ public int count(LeafReaderContext context) throws IOException {
             return in.count(context);
         }
 
-        @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            shardKeyMap.add(context.reader());
-            return in.scorer(context);
-        }
-
         @Override
         public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
             shardKeyMap.add(context.reader());
             return in.scorerSupplier(context);
         }
 
-        @Override
-        public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
-            shardKeyMap.add(context.reader());
-            return in.bulkScorer(context);
-        }
-
         @Override
         public boolean isCacheable(LeafReaderContext ctx) {
             return in.isCacheable(ctx);
diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java
index 08148de0591cb..f3456870114f5 100644
--- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java
+++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java
@@ -41,6 +41,8 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
+
 /**
  * Uses a pattern string to define a protected space for indices belonging to a system feature, and, if needed, provides metadata for
  * managing indices that match the pattern.
@@ -360,7 +362,7 @@ protected SystemIndexDescriptor(
         this.primaryIndex = primaryIndex;
         this.aliasName = aliasName;
 
-        final Automaton automaton = buildAutomaton(indexPattern, aliasName);
+        final Automaton automaton = Operations.determinize(buildAutomaton(indexPattern, aliasName), DEFAULT_DETERMINIZE_WORK_LIMIT);
         this.indexPatternAutomaton = new CharacterRunAutomaton(automaton);
         if (primaryIndex != null && indexPatternAutomaton.run(primaryIndex) == false) {
             throw new IllegalArgumentException("primary index does not match the index pattern!");
@@ -883,15 +885,15 @@ static Automaton buildAutomaton(String pattern, String alias) {
         final String patternAsRegex = patternToRegex(pattern);
         final String aliasAsRegex = alias == null ? null : patternToRegex(alias);
 
-        final Automaton patternAutomaton = new RegExp(patternAsRegex).toAutomaton();
+        final Automaton patternAutomaton = new RegExp(patternAsRegex, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton();
 
         if (aliasAsRegex == null) {
             return patternAutomaton;
         }
 
-        final Automaton aliasAutomaton = new RegExp(aliasAsRegex).toAutomaton();
+        final Automaton aliasAutomaton = new RegExp(aliasAsRegex, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton();
 
-        return Operations.union(patternAutomaton, aliasAutomaton);
+        return Operations.determinize(Operations.union(patternAutomaton, aliasAutomaton), DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
index a0a4388a4d54a..856b30d1c19e8 100644
--- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
+++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
@@ -14,7 +14,6 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus;
@@ -178,7 +177,7 @@ public SystemIndices(List<Feature> pluginAndModuleFeatures) {
         this.netNewSystemIndexAutomaton = buildNetNewIndexCharacterRunAutomaton(featureDescriptors);
         this.productToSystemIndicesMatcher = getProductToSystemIndicesMap(featureDescriptors);
         this.executorSelector = new ExecutorSelector(this);
-        this.systemNameAutomaton = MinimizationOperations.minimize(
+        this.systemNameAutomaton = Operations.determinize(
             Operations.union(List.of(systemIndexAutomata, systemDataStreamIndicesAutomata, buildDataStreamAutomaton(featureDescriptors))),
             Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
         );
@@ -264,9 +263,7 @@ private static Map<String, CharacterRunAutomaton> getProductToSystemIndicesMap(M
             .collect(
                 Collectors.toUnmodifiableMap(
                     Entry::getKey,
-                    entry -> new CharacterRunAutomaton(
-                        MinimizationOperations.minimize(entry.getValue(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
-                    )
+                    entry -> new CharacterRunAutomaton(Operations.determinize(entry.getValue(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT))
                 )
             );
     }
@@ -426,7 +423,7 @@ private static Automaton buildIndexAutomaton(Map<String, Feature> featureDescrip
             .stream()
             .map(SystemIndices::featureToIndexAutomaton)
             .reduce(Operations::union);
-        return MinimizationOperations.minimize(automaton.orElse(EMPTY), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        return Operations.determinize(automaton.orElse(EMPTY), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     private static CharacterRunAutomaton buildNetNewIndexCharacterRunAutomaton(Map<String, Feature> featureDescriptors) {
@@ -437,9 +434,7 @@ private static CharacterRunAutomaton buildNetNewIndexCharacterRunAutomaton(Map<S
             .filter(SystemIndexDescriptor::isNetNew)
             .map(descriptor -> SystemIndexDescriptor.buildAutomaton(descriptor.getIndexPattern(), descriptor.getAliasName()))
             .reduce(Operations::union);
-        return new CharacterRunAutomaton(
-            MinimizationOperations.minimize(automaton.orElse(EMPTY), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
-        );
+        return new CharacterRunAutomaton(Operations.determinize(automaton.orElse(EMPTY), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT));
     }
 
     private static Automaton featureToIndexAutomaton(Feature feature) {
@@ -459,7 +454,7 @@ private static Automaton buildDataStreamAutomaton(Map<String, Feature> featureDe
             .map(dsName -> SystemIndexDescriptor.buildAutomaton(dsName, null))
             .reduce(Operations::union);
 
-        return automaton.isPresent() ? MinimizationOperations.minimize(automaton.get(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT) : EMPTY;
+        return automaton.isPresent() ? Operations.determinize(automaton.get(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT) : EMPTY;
     }
 
     private static Predicate<String> buildDataStreamNamePredicate(Map<String, Feature> featureDescriptors) {
@@ -472,7 +467,7 @@ private static Automaton buildDataStreamBackingIndicesAutomaton(Map<String, Feat
             .stream()
             .map(SystemIndices::featureToDataStreamBackingIndicesAutomaton)
             .reduce(Operations::union);
-        return MinimizationOperations.minimize(automaton.orElse(EMPTY), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        return Operations.determinize(automaton.orElse(EMPTY), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     private static Automaton featureToDataStreamBackingIndicesAutomaton(Feature feature) {
diff --git a/server/src/main/java/org/elasticsearch/indices/TermsLookup.java b/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
index b1c74c606df64..0bab521a984e2 100644
--- a/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
+++ b/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -25,7 +24,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.core.RestApiVersion.equalTo;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 
 /**
@@ -109,7 +107,6 @@ public TermsLookup routing(String routing) {
         PARSER.declareString(constructorArg(), new ParseField("id"));
         PARSER.declareString(constructorArg(), new ParseField("path"));
         PARSER.declareString(TermsLookup::routing, new ParseField("routing"));
-        PARSER.declareString((termLookup, type) -> {}, new ParseField("type").forRestApiVersion(equalTo(RestApiVersion.V_7)));
     }
 
     public static TermsLookup parseTermsLookup(XContentParser parser) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 556bf43425bd8..fbfed8a75a146 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -1376,7 +1376,7 @@ protected void onNewResource(StoreFileMetadata md) throws IOException {
                         // we already have the file contents on heap no need to open the file again
                         currentInput = null;
                     } else {
-                        currentInput = store.directory().openInput(md.name(), IOContext.READ);
+                        currentInput = store.directory().openInput(md.name(), IOContext.DEFAULT);
                     }
                 }
 
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
index 6be94ab21a4f7..b0d33a75ba883 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
@@ -293,9 +293,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.TYPE, recoverySource.getType());
         builder.field(Fields.STAGE, stage.toString());
         builder.field(Fields.PRIMARY, primary);
-        builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, timer.startTime);
+        builder.timestampFieldsFromUnixEpochMillis(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, timer.startTime);
         if (timer.stopTime > 0) {
-            builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, timer.stopTime);
+            builder.timestampFieldsFromUnixEpochMillis(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, timer.stopTime);
         }
         builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(timer.time()));
 
diff --git a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java
index 0e5b3a555b800..9c666bd4a35f5 100644
--- a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java
+++ b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * This class defines an empty secret settings object. This is useful for services that do not have any secret settings.
@@ -48,4 +49,9 @@ public TransportVersion getMinimalSupportedVersion() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {}
+
+    @Override
+    public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+        return INSTANCE;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java
index 0c863932c6afe..cba0282f7fed8 100644
--- a/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java
+++ b/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * This class defines an empty task settings object. This is useful for services that do not have any task settings.
@@ -53,4 +54,9 @@ public TransportVersion getMinimalSupportedVersion() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {}
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        return INSTANCE;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java
index cbbfef2cc65fa..2c99563955746 100644
--- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java
+++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java
@@ -129,10 +129,10 @@ void chunkedInfer(
     /**
      * Stop the model deployment.
      * The default action does nothing except acknowledge the request (true).
-     * @param modelId The ID of the model to be stopped
+     * @param unparsedModel The unparsed model configuration
      * @param listener The listener
      */
-    default void stop(String modelId, ActionListener<Boolean> listener) {
+    default void stop(UnparsedModel unparsedModel, ActionListener<Boolean> listener) {
         listener.onResponse(true);
     }
 
@@ -192,12 +192,26 @@ default boolean canStream(TaskType taskType) {
         return supportedStreamingTasks().contains(taskType);
     }
 
+    record DefaultConfigId(String inferenceId, TaskType taskType, InferenceService service) {};
+
     /**
-     * A service can define default configurations that can be
-     * used out of the box without creating an endpoint first.
-     * @return Default configurations provided by this service
+     * Get the Ids and task type of any default configurations provided by this service
+     * @return Defaults
      */
-    default List<UnparsedModel> defaultConfigs() {
+    default List<DefaultConfigId> defaultConfigIds() {
         return List.of();
     }
+
+    /**
+     * Call the listener with the default model configurations defined by
+     * the service
+     * @param defaultsListener The listener
+     */
+    default void defaultConfigs(ActionListener<List<Model>> defaultsListener) {
+        defaultsListener.onResponse(List.of());
+    }
+
+    default void updateModelsWithDynamicFields(List<Model> model, ActionListener<List<Model>> listener) {
+        listener.onResponse(model);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java
index 68dc865b4c7db..3274bf571d10a 100644
--- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java
+++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java
@@ -10,6 +10,8 @@
 package org.elasticsearch.inference;
 
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.List;
@@ -21,7 +23,7 @@ public interface InferenceServiceExtension {
 
     List<Factory> getInferenceServiceFactories();
 
-    record InferenceServiceFactoryContext(Client client, ThreadPool threadPool) {}
+    record InferenceServiceFactoryContext(Client client, ThreadPool threadPool, ClusterService clusterService, Settings settings) {}
 
     interface Factory {
         /**
diff --git a/server/src/main/java/org/elasticsearch/inference/SecretSettings.java b/server/src/main/java/org/elasticsearch/inference/SecretSettings.java
index e2c0c8b58c69b..90ca92bb0e2ef 100644
--- a/server/src/main/java/org/elasticsearch/inference/SecretSettings.java
+++ b/server/src/main/java/org/elasticsearch/inference/SecretSettings.java
@@ -12,6 +12,9 @@
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.xcontent.ToXContentObject;
 
+import java.util.Map;
+
 public interface SecretSettings extends ToXContentObject, VersionedNamedWriteable {
 
+    SecretSettings newSecretSettings(Map<String, Object> newSecrets);
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/TaskSettings.java b/server/src/main/java/org/elasticsearch/inference/TaskSettings.java
index 9862abce2332c..7dd20688245ba 100644
--- a/server/src/main/java/org/elasticsearch/inference/TaskSettings.java
+++ b/server/src/main/java/org/elasticsearch/inference/TaskSettings.java
@@ -12,6 +12,11 @@
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.xcontent.ToXContentObject;
 
+import java.util.Map;
+
 public interface TaskSettings extends ToXContentObject, VersionedNamedWriteable {
+
     boolean isEmpty();
+
+    TaskSettings updatedTaskSettings(Map<String, Object> newSettings);
 }
diff --git a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 5059272aa2e23..97a68d9807688 100644
--- a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -239,7 +240,7 @@ private static Boolean readBoolean(String processorType, String processorTag, St
             processorType,
             processorTag,
             propertyName,
-            "property isn't a boolean, but of type [" + value.getClass().getName() + "]"
+            Strings.format("property isn't a boolean, but of type [%s]", value.getClass().getName())
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java
index 1933d285d7870..77b11357d79b1 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java
@@ -22,7 +22,13 @@ public class IngestGeoIpFeatures implements FeatureSpecification {
         "get_database_configuration_action.multi_node"
     );
 
+    public static final NodeFeature PUT_DATABASE_CONFIGURATION_ACTION_IPINFO = new NodeFeature("put_database_configuration_action.ipinfo");
+
     public Set<NodeFeature> getFeatures() {
-        return Set.of(GEOIP_DOWNLOADER_DATABASE_CONFIGURATION, GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE);
+        return Set.of(
+            GEOIP_DOWNLOADER_DATABASE_CONFIGURATION,
+            GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE,
+            PUT_DATABASE_CONFIGURATION_ACTION_IPINFO
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
index 05da1f4784649..316f621e80669 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -98,8 +98,8 @@ public static IngestMetadata fromXContent(XContentParser parser) throws IOExcept
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
-        return ChunkedToXContentHelper.array(PIPELINES_FIELD.getPreferredName(), pipelines.values().iterator());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContent.builder(params).array(PIPELINES_FIELD.getPreferredName(), pipelines.values().iterator());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
index 0f63d2a8dcc1b..99ff44a3cd135 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -46,6 +47,8 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.TriConsumer;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -55,7 +58,9 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.grok.MatcherWatchdog;
 import org.elasticsearch.index.IndexSettings;
@@ -97,6 +102,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.core.UpdateForV10.Owner.DATA_MANAGEMENT;
 
 /**
  * Holder class for several ingest related services.
@@ -107,7 +113,10 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
 
     public static final String INGEST_ORIGIN = "ingest";
 
+    public static final NodeFeature PIPELINE_NAME_VALIDATION_WARNINGS = new NodeFeature("ingest.pipeline_name_special_chars_warning");
+
     private static final Logger logger = LogManager.getLogger(IngestService.class);
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IngestService.class);
 
     private final MasterServiceTaskQueue<PipelineClusterStateUpdateTask> taskQueue;
     private final ClusterService clusterService;
@@ -652,12 +661,24 @@ public IngestMetadata execute(IngestMetadata currentIngestMetadata, Collection<I
         }
     }
 
+    @UpdateForV10(owner = DATA_MANAGEMENT) // Change deprecation log for special characters in name to a failure
     void validatePipeline(Map<DiscoveryNode, IngestInfo> ingestInfos, String pipelineId, Map<String, Object> pipelineConfig)
         throws Exception {
         if (ingestInfos.isEmpty()) {
             throw new IllegalStateException("Ingest info is empty");
         }
 
+        try {
+            MetadataCreateIndexService.validateIndexOrAliasName(
+                pipelineId,
+                (pipelineName, error) -> new IllegalArgumentException(
+                    "Pipeline name [" + pipelineName + "] will be disallowed in a future version for the following reason: " + error
+                )
+            );
+        } catch (IllegalArgumentException e) {
+            deprecationLogger.critical(DeprecationCategory.API, "pipeline_name_special_chars", e.getMessage());
+        }
+
         Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService);
         List<Exception> exceptions = new ArrayList<>();
         for (Processor processor : pipeline.flattenAllProcessors()) {
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java
index a7f0a16b9025d..39df4e53423a2 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java
@@ -65,9 +65,7 @@ public static IngestStats read(StreamInput in) throws IOException {
         for (var i = 0; i < size; i++) {
             var pipelineId = in.readString();
             var pipelineStat = new Stats(in);
-            var byteStat = in.getTransportVersion().onOrAfter(TransportVersions.NODE_STATS_INGEST_BYTES)
-                ? new ByteStats(in)
-                : new ByteStats(0, 0);
+            var byteStat = in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? new ByteStats(in) : new ByteStats(0, 0);
             pipelineStats.add(new PipelineStat(pipelineId, pipelineStat, byteStat));
             int processorsSize = in.readVInt();
             var processorStatsPerPipeline = new ArrayList<ProcessorStat>(processorsSize);
@@ -90,7 +88,7 @@ public void writeTo(StreamOutput out) throws IOException {
         for (PipelineStat pipelineStat : pipelineStats) {
             out.writeString(pipelineStat.pipelineId());
             pipelineStat.stats().writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.NODE_STATS_INGEST_BYTES)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 pipelineStat.byteStats().writeTo(out);
             }
             List<ProcessorStat> processorStatsForPipeline = processorStats.get(pipelineStat.pipelineId());
diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
index de172d86b810d..7406ee8837264 100644
--- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
+++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
@@ -99,18 +99,13 @@ BytesReference getConfig() {
     }
 
     public Integer getVersion() {
-        var configMap = getConfigAsMap();
-        if (configMap.containsKey("version")) {
-            Object o = configMap.get("version");
-            if (o == null) {
-                return null;
-            } else if (o instanceof Number number) {
-                return number.intValue();
-            } else {
-                throw new IllegalStateException("unexpected version type [" + o.getClass().getName() + "]");
-            }
-        } else {
+        Object o = getConfigAsMap().get("version");
+        if (o == null) {
             return null;
+        } else if (o instanceof Number number) {
+            return number.intValue();
+        } else {
+            throw new IllegalStateException("unexpected version type [" + o.getClass().getName() + "]");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/internal/BuildExtension.java b/server/src/main/java/org/elasticsearch/internal/BuildExtension.java
index a23270cb5550c..427e186bc40cf 100644
--- a/server/src/main/java/org/elasticsearch/internal/BuildExtension.java
+++ b/server/src/main/java/org/elasticsearch/internal/BuildExtension.java
@@ -38,4 +38,9 @@ default boolean hasReleaseVersioning() {
      * Returns the {@link BuildVersion} for a given version identifier.
      */
     BuildVersion fromVersionId(int versionId);
+
+    /**
+     * Returns the {@link BuildVersion} for a given version string.
+     */
+    BuildVersion fromString(String version);
 }
diff --git a/server/src/main/java/org/elasticsearch/lucene/grouping/GroupingDocValuesSelector.java b/server/src/main/java/org/elasticsearch/lucene/grouping/GroupingDocValuesSelector.java
index 49041566a8574..fe076c7155c8e 100644
--- a/server/src/main/java/org/elasticsearch/lucene/grouping/GroupingDocValuesSelector.java
+++ b/server/src/main/java/org/elasticsearch/lucene/grouping/GroupingDocValuesSelector.java
@@ -199,12 +199,12 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException {
                             @Override
                             public boolean advanceExact(int target) throws IOException {
                                 if (sorted.advanceExact(target)) {
-                                    ord = (int) sorted.nextOrd();
-                                    if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
+                                    if (sorted.docValueCount() > 1) {
                                         throw new IllegalStateException(
                                             "failed to extract doc:" + target + ", the grouping field must be single valued"
                                         );
                                     }
+                                    ord = (int) sorted.nextOrd();
                                     return true;
                                 } else {
                                     return false;
diff --git a/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java b/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java
index ed07525c1dd7b..443963dd59dcd 100644
--- a/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java
+++ b/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java
@@ -170,10 +170,10 @@ public static TopFieldGroups merge(Sort sort, int start, int size, TopFieldGroup
             final TopFieldGroups shard = shardHits[shardIDX];
             // totalHits can be non-zero even if no hits were
             // collected, when searchAfter was used:
-            totalHitCount += shard.totalHits.value;
+            totalHitCount += shard.totalHits.value();
             // If any hit count is a lower bound then the merged
             // total hit count is a lower bound as well
-            if (shard.totalHits.relation == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
+            if (shard.totalHits.relation() == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
                 totalHitsRelation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
             }
             if (CollectionUtils.isEmpty(shard.scoreDocs) == false) {
diff --git a/server/src/main/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQuery.java
index dca4ff503c788..67ece200c06ee 100644
--- a/server/src/main/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQuery.java
+++ b/server/src/main/java/org/elasticsearch/lucene/queries/BinaryDocValuesRangeQuery.java
@@ -18,7 +18,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BytesRef;
@@ -61,7 +61,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
         return new ConstantScoreWeight(this, boost) {
 
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 final BinaryDocValues values = context.reader().getBinaryDocValues(fieldName);
                 if (values == null) {
                     return null;
@@ -106,7 +106,8 @@ public float matchCost() {
                         return 4; // at most 4 comparisons
                     }
                 };
-                return new ConstantScoreScorer(this, score(), scoreMode, iterator);
+
+                return new DefaultScorerSupplier(new ConstantScoreScorer(score(), scoreMode, iterator));
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java
index c75c6e2373f25..788bf76087d1f 100644
--- a/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java
+++ b/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.IOSupplier;
 import org.apache.lucene.util.InPlaceMergeSorter;
 
 import java.io.IOException;
@@ -188,7 +189,11 @@ private static TermStates adjustTTF(IndexReaderContext readerContext, TermStates
         int df = termContext.docFreq();
         long ttf = sumTTF;
         for (int i = 0; i < len; i++) {
-            TermState termState = termContext.get(leaves.get(i));
+            IOSupplier<TermState> termStateSupplier = termContext.get(leaves.get(i));
+            if (termStateSupplier == null) {
+                continue;
+            }
+            TermState termState = termStateSupplier.get();
             if (termState == null) {
                 continue;
             }
@@ -212,7 +217,11 @@ private static TermStates adjustDF(IndexReaderContext readerContext, TermStates
         }
         TermStates newCtx = new TermStates(readerContext);
         for (int i = 0; i < len; ++i) {
-            TermState termState = ctx.get(leaves.get(i));
+            IOSupplier<TermState> termStateSupplier = ctx.get(leaves.get(i));
+            if (termStateSupplier == null) {
+                continue;
+            }
+            TermState termState = termStateSupplier.get();
             if (termState == null) {
                 continue;
             }
diff --git a/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java
index 8e85c1d974382..13b0bf650a39e 100644
--- a/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java
+++ b/server/src/main/java/org/elasticsearch/lucene/queries/MinDocQuery.java
@@ -19,6 +19,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 
 import java.io.IOException;
@@ -76,15 +77,17 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
             throw new IllegalStateException("Executing against a different reader than the query has been rewritten against");
         }
         return new ConstantScoreWeight(this, boost) {
+
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 final int maxDoc = context.reader().maxDoc();
                 if (context.docBase + maxDoc <= minDoc) {
                     return null;
                 }
                 final int segmentMinDoc = Math.max(0, minDoc - context.docBase);
                 final DocIdSetIterator disi = new MinDocIterator(segmentMinDoc, maxDoc);
-                return new ConstantScoreScorer(this, score(), scoreMode, disi);
+                Scorer scorer = new ConstantScoreScorer(score(), scoreMode, disi);
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java
index 1a8ac203f0cb5..6575f7f416bd9 100644
--- a/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java
+++ b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.Weight;
@@ -67,8 +68,8 @@ public SearchAfterSortedDocQuery(Sort sort, FieldDoc after) {
     public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
         return new ConstantScoreWeight(this, 1.0f) {
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                Sort segmentSort = context.reader().getMetaData().getSort();
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                Sort segmentSort = context.reader().getMetaData().sort();
                 if (segmentSort == null || Lucene.canEarlyTerminate(sort, segmentSort) == false) {
                     throw new IOException("search sort :[" + sort + "] does not match the index sort:[" + segmentSort + "]");
                 }
@@ -80,7 +81,8 @@ public Scorer scorer(LeafReaderContext context) throws IOException {
                     return null;
                 }
                 final DocIdSetIterator disi = new MinDocQuery.MinDocIterator(firstDoc, maxDoc);
-                return new ConstantScoreScorer(this, score(), scoreMode, disi);
+                Scorer scorer = new ConstantScoreScorer(score(), scoreMode, disi);
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java b/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java
index bd64ee88fc300..064f8ef3eacd8 100644
--- a/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java
+++ b/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java
@@ -109,14 +109,8 @@ private ConstantScoreWeight getStandardWeight(ScoreMode scoreMode, float boost)
         final Component2D component2D = create(geometries);
         return new ConstantScoreWeight(this, boost) {
 
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                return scorerSupplier(context).get(Long.MAX_VALUE);
-            }
-
             @Override
             public ScorerSupplier scorerSupplier(LeafReaderContext context) {
-                final Weight weight = this;
                 // implement ScorerSupplier, since we do some expensive stuff to make a scorer
                 return new ScorerSupplier() {
 
@@ -125,7 +119,7 @@ public Scorer get(long leadCost) throws IOException {
                         // binary doc values allocate an array upfront, lets only allocate it if we are going to use it
                         final BinaryDocValues values = context.reader().getBinaryDocValues(field);
                         if (values == null) {
-                            return new ConstantScoreScorer(weight, 0f, scoreMode, DocIdSetIterator.empty());
+                            return new ConstantScoreScorer(0f, scoreMode, DocIdSetIterator.empty());
                         }
                         final GeometryDocValueReader reader = new GeometryDocValueReader();
                         final Component2DVisitor visitor = Component2DVisitor.getVisitor(component2D, relation, encoder);
@@ -143,7 +137,7 @@ public float matchCost() {
                                 return 1000f; // TODO: what should it be?
                             }
                         };
-                        return new ConstantScoreScorer(weight, score(), scoreMode, iterator);
+                        return new ConstantScoreScorer(score(), scoreMode, iterator);
                     }
 
                     @Override
@@ -167,14 +161,8 @@ private ConstantScoreWeight getContainsWeight(ScoreMode scoreMode, float boost)
         }
         return new ConstantScoreWeight(this, boost) {
 
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                return scorerSupplier(context).get(Long.MAX_VALUE);
-            }
-
             @Override
             public ScorerSupplier scorerSupplier(LeafReaderContext context) {
-                final Weight weight = this;
                 // implement ScorerSupplier, since we do some expensive stuff to make a scorer
                 return new ScorerSupplier() {
 
@@ -183,7 +171,7 @@ public Scorer get(long leadCost) throws IOException {
                         // binary doc values allocate an array upfront, lets only allocate it if we are going to use it
                         final BinaryDocValues values = context.reader().getBinaryDocValues(field);
                         if (values == null) {
-                            return new ConstantScoreScorer(weight, 0f, scoreMode, DocIdSetIterator.empty());
+                            return new ConstantScoreScorer(0f, scoreMode, DocIdSetIterator.empty());
                         }
                         final Component2DVisitor[] visitors = new Component2DVisitor[components2D.size()];
                         for (int i = 0; i < components2D.size(); i++) {
@@ -210,7 +198,7 @@ public float matchCost() {
                                 return 1000f; // TODO: what should it be?
                             }
                         };
-                        return new ConstantScoreScorer(weight, score(), scoreMode, iterator);
+                        return new ConstantScoreScorer(score(), scoreMode, iterator);
                     }
 
                     @Override
diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/GcNames.java b/server/src/main/java/org/elasticsearch/monitor/jvm/GcNames.java
index 9db8e8f414d5c..3494204c330c0 100644
--- a/server/src/main/java/org/elasticsearch/monitor/jvm/GcNames.java
+++ b/server/src/main/java/org/elasticsearch/monitor/jvm/GcNames.java
@@ -15,8 +15,14 @@ public class GcNames {
     public static final String OLD = "old";
     public static final String SURVIVOR = "survivor";
 
+    private GcNames() {}
+
     /**
-     * Resolves the GC type by its memory pool name ({@link java.lang.management.MemoryPoolMXBean#getName()}.
+     * Resolves the memory area name by the memory pool name provided by {@link java.lang.management.MemoryPoolMXBean#getName()}
+     *
+     * @param poolName the name of the memory pool from {@link java.lang.management.MemoryPoolMXBean}
+     * @param defaultName the name to return if the pool name does not match any known memory area
+     * @return memory area name corresponding to the pool name or {@code defaultName} if no match is found
      */
     public static String getByMemoryPoolName(String poolName, String defaultName) {
         if ("Eden Space".equals(poolName)
@@ -40,6 +46,13 @@ public static String getByMemoryPoolName(String poolName, String defaultName) {
         return defaultName;
     }
 
+    /**
+     * Resolves the GC type by the GC name provided by {@link java.lang.management.GarbageCollectorMXBean#getName()}
+     *
+     * @param gcName the name of the GC from {@link java.lang.management.GarbageCollectorMXBean}
+     * @param defaultName the name to return if the GC name does not match any known GC type
+     * @return GC type corresponding to the GC name or {@code defaultName} if no match is found
+     */
     public static String getByGcName(String gcName, String defaultName) {
         if ("Copy".equals(gcName) || "PS Scavenge".equals(gcName) || "ParNew".equals(gcName) || "G1 Young Generation".equals(gcName)) {
             return YOUNG;
diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
index 1c68615203d3a..a3639214a1b9d 100644
--- a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
+++ b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
@@ -420,7 +420,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.VM_VERSION, vmVersion);
         builder.field(Fields.VM_VENDOR, vmVendor);
         builder.field(Fields.USING_BUNDLED_JDK, usingBundledJdk);
-        builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, startTime);
+        builder.timestampFieldsFromUnixEpochMillis(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, startTime);
 
         builder.startObject(Fields.MEM);
         builder.humanReadableField(Fields.HEAP_INIT_IN_BYTES, Fields.HEAP_INIT, ByteSizeValue.ofBytes(mem.heapInit));
diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
index 0a2763474b8df..e6b109207fdf3 100644
--- a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
+++ b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
@@ -64,10 +64,7 @@ public static JvmStats jvmStats() {
         List<MemoryPool> pools = new ArrayList<>();
         for (MemoryPoolMXBean memoryPoolMXBean : memoryPoolMXBeans) {
             try {
-                String name = GcNames.getByMemoryPoolName(memoryPoolMXBean.getName(), null);
-                if (name == null) { // if we can't resolve it, its not interesting.... (Per Gen, Code Cache)
-                    continue;
-                }
+                String name = GcNames.getByMemoryPoolName(memoryPoolMXBean.getName(), memoryPoolMXBean.getName());
                 MemoryUsage usage = memoryPoolMXBean.getUsage();
                 MemoryUsage peakUsage = memoryPoolMXBean.getPeakUsage();
                 pools.add(
diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
index 11df8710fad6c..ba67bc03e1441 100644
--- a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
+++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
@@ -55,7 +55,7 @@ public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService
     }
 
     private static List<AutoCloseable> registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) {
-        final int TOTAL_METRICS = 36;
+        final int TOTAL_METRICS = 48;
         List<AutoCloseable> metrics = new ArrayList<>(TOTAL_METRICS);
         for (IndexMode indexMode : IndexMode.values()) {
             String name = indexMode.getName();
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 5024cc5468866..ec4a534fc883b 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -13,10 +13,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchTimeoutException;
-import org.elasticsearch.action.search.TransportSearchAction;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.RefCountingListener;
-import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.bootstrap.BootstrapContext;
 import org.elasticsearch.client.internal.Client;
@@ -82,7 +78,6 @@
 import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.TaskCancellationService;
-import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterPortSettings;
@@ -106,18 +101,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
 
 import javax.net.ssl.SNIHostName;
 
-import static org.elasticsearch.core.Strings.format;
-
 /**
  * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used
  * in order to use a {@link Client} to perform actions/operations against the cluster.
@@ -161,12 +150,6 @@ public class Node implements Closeable {
         Property.NodeScope
     );
 
-    public static final Setting<TimeValue> MAXIMUM_SHUTDOWN_TIMEOUT_SETTING = Setting.positiveTimeSetting(
-        "node.maximum_shutdown_grace_period",
-        TimeValue.ZERO,
-        Setting.Property.NodeScope
-    );
-
     private final Lifecycle lifecycle = new Lifecycle();
 
     /**
@@ -187,6 +170,7 @@ public class Node implements Closeable {
     private final LocalNodeFactory localNodeFactory;
     private final NodeService nodeService;
     private final TerminationHandler terminationHandler;
+
     // for testing
     final NamedWriteableRegistry namedWriteableRegistry;
     final NamedXContentRegistry namedXContentRegistry;
@@ -341,7 +325,6 @@ public Node start() throws NodeValidationException {
         // TODO: Do not expect that the legacy metadata file is always present https://github.com/elastic/elasticsearch/issues/95211
         if (Assertions.ENABLED && DiscoveryNode.isStateless(settings()) == false) {
             try {
-                assert injector.getInstance(MetaStateService.class).loadFullState().v1().isEmpty();
                 final NodeMetadata nodeMetadata = NodeMetadata.FORMAT.loadLatestState(
                     logger,
                     NamedXContentRegistry.EMPTY,
@@ -561,7 +544,7 @@ public synchronized void close() throws IOException {
         toClose.add(() -> stopWatch.stop().start("transport"));
         toClose.add(injector.getInstance(TransportService.class));
         toClose.add(injector.getInstance(NodeMetrics.class));
-        toClose.add(injector.getInstance(IndicesService.class));
+        toClose.add(injector.getInstance(IndicesMetrics.class));
         if (ReadinessService.enabled(environment)) {
             toClose.add(injector.getInstance(ReadinessService.class));
         }
@@ -606,105 +589,8 @@ public synchronized void close() throws IOException {
      * logic should use Node Shutdown, see {@link org.elasticsearch.cluster.metadata.NodesShutdownMetadata}.
      */
     public void prepareForClose() {
-        final var maxTimeout = MAXIMUM_SHUTDOWN_TIMEOUT_SETTING.get(this.settings());
-
-        record Stopper(String name, SubscribableListener<Void> listener) {
-            boolean isIncomplete() {
-                return listener().isDone() == false;
-            }
-        }
-
-        final var stoppers = new ArrayList<Stopper>();
-        final var allStoppersFuture = new PlainActionFuture<Void>();
-        try (var listeners = new RefCountingListener(allStoppersFuture)) {
-            final BiConsumer<String, Runnable> stopperRunner = (name, action) -> {
-                final var stopper = new Stopper(name, new SubscribableListener<>());
-                stoppers.add(stopper);
-                stopper.listener().addListener(listeners.acquire());
-                new Thread(() -> {
-                    try {
-                        action.run();
-                    } catch (Exception ex) {
-                        logger.warn("unexpected exception in shutdown task [" + stopper.name() + "]", ex);
-                    } finally {
-                        stopper.listener().onResponse(null);
-                    }
-                }, stopper.name()).start();
-            };
-
-            stopperRunner.accept("http-server-transport-stop", injector.getInstance(HttpServerTransport.class)::close);
-            stopperRunner.accept("async-search-stop", () -> awaitSearchTasksComplete(maxTimeout));
-            if (terminationHandler != null) {
-                stopperRunner.accept("termination-handler-stop", terminationHandler::handleTermination);
-            }
-        }
-
-        final Supplier<String> incompleteStoppersDescriber = () -> stoppers.stream()
-            .filter(Stopper::isIncomplete)
-            .map(Stopper::name)
-            .collect(Collectors.joining(", ", "[", "]"));
-
-        try {
-            if (TimeValue.ZERO.equals(maxTimeout)) {
-                allStoppersFuture.get();
-            } else {
-                allStoppersFuture.get(maxTimeout.millis(), TimeUnit.MILLISECONDS);
-            }
-        } catch (ExecutionException e) {
-            assert false : e; // listeners are never completed exceptionally
-            logger.warn("failed during graceful shutdown tasks", e);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            logger.warn("interrupted while waiting for graceful shutdown tasks: " + incompleteStoppersDescriber.get(), e);
-        } catch (TimeoutException e) {
-            logger.warn("timed out while waiting for graceful shutdown tasks: " + incompleteStoppersDescriber.get());
-        }
-    }
-
-    private void awaitSearchTasksComplete(TimeValue asyncSearchTimeout) {
-        TaskManager taskManager = injector.getInstance(TransportService.class).getTaskManager();
-        long millisWaited = 0;
-        while (true) {
-            long searchTasksRemaining = taskManager.getTasks()
-                .values()
-                .stream()
-                .filter(task -> TransportSearchAction.TYPE.name().equals(task.getAction()))
-                .count();
-            if (searchTasksRemaining == 0) {
-                logger.debug("all search tasks complete");
-                return;
-            } else {
-                // Let the system work on those searches for a while. We're on a dedicated thread to manage app shutdown, so we
-                // literally just want to wait and not take up resources on this thread for now. Poll period chosen to allow short
-                // response times, but checking the tasks list is relatively expensive, and we don't want to waste CPU time we could
-                // be spending on finishing those searches.
-                final TimeValue pollPeriod = TimeValue.timeValueMillis(500);
-                millisWaited += pollPeriod.millis();
-                if (TimeValue.ZERO.equals(asyncSearchTimeout) == false && millisWaited >= asyncSearchTimeout.millis()) {
-                    logger.warn(
-                        format(
-                            "timed out after waiting [%s] for [%d] search tasks to finish",
-                            asyncSearchTimeout.toString(),
-                            searchTasksRemaining
-                        )
-                    );
-                    return;
-                }
-                logger.debug(format("waiting for [%s] search tasks to finish, next poll in [%s]", searchTasksRemaining, pollPeriod));
-                try {
-                    Thread.sleep(pollPeriod.millis());
-                } catch (InterruptedException ex) {
-                    logger.warn(
-                        format(
-                            "interrupted while waiting [%s] for [%d] search tasks to finish",
-                            asyncSearchTimeout.toString(),
-                            searchTasksRemaining
-                        )
-                    );
-                    return;
-                }
-            }
-        }
+        injector.getInstance(ShutdownPrepareService.class)
+            .prepareForShutdown(injector.getInstance(TransportService.class).getTaskManager());
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index 9b33ac2ea12fc..784e02059823b 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -80,6 +80,7 @@
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
@@ -108,6 +109,7 @@
 import org.elasticsearch.health.node.tracker.RepositoriesHealthTracker;
 import org.elasticsearch.health.stats.HealthApiStats;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.index.IndexingPressure;
@@ -415,8 +417,8 @@ private Settings createEnvironment(Environment initialEnvironment, NodeServicePr
             Constants.OS_ARCH,
             Constants.JVM_VENDOR,
             Constants.JVM_NAME,
-            Constants.JAVA_VERSION,
-            Constants.JVM_VERSION
+            System.getProperty("java.version"),
+            Runtime.version().toString()
         );
         logger.info("JVM home [{}], using bundled JDK [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk());
         logger.info("JVM arguments {}", Arrays.toString(jvmInfo.getInputArguments()));
@@ -820,7 +822,10 @@ private void construct(
 
         final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation);
         IndexSettingProviders indexSettingProviders = new IndexSettingProviders(
-            pluginsService.flatMap(p -> p.getAdditionalIndexSettingProviders(parameters)).collect(Collectors.toSet())
+            Sets.union(
+                builtinIndexSettingProviders(),
+                pluginsService.flatMap(p -> p.getAdditionalIndexSettingProviders(parameters)).collect(Collectors.toSet())
+            )
         );
 
         final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService);
@@ -1099,6 +1104,8 @@ private void construct(
             telemetryProvider.getTracer()
         );
 
+        final ShutdownPrepareService shutdownPrepareService = new ShutdownPrepareService(settings, httpServerTransport, terminationHandler);
+
         modules.add(
             loadPersistentTasksService(
                 settingsModule,
@@ -1200,6 +1207,7 @@ private void construct(
             b.bind(CompatibilityVersions.class).toInstance(compatibilityVersions);
             b.bind(DataStreamAutoShardingService.class).toInstance(dataStreamAutoShardingService);
             b.bind(FailureStoreMetrics.class).toInstance(failureStoreMetrics);
+            b.bind(ShutdownPrepareService.class).toInstance(shutdownPrepareService);
         });
 
         if (ReadinessService.enabled(environment)) {
@@ -1653,4 +1661,7 @@ private Module loadPersistentTasksService(
         };
     }
 
+    private Set<IndexSettingProvider> builtinIndexSettingProviders() {
+        return Set.of(new IndexMode.IndexModeSettingsProvider());
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/node/ShutdownPrepareService.java b/server/src/main/java/org/elasticsearch/node/ShutdownPrepareService.java
new file mode 100644
index 0000000000000..ab9537053f45d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/node/ShutdownPrepareService.java
@@ -0,0 +1,184 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.node;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.index.reindex.ReindexAction;
+import org.elasticsearch.node.internal.TerminationHandler;
+import org.elasticsearch.tasks.TaskManager;
+
+import java.util.ArrayList;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BiConsumer;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.core.Strings.format;
+
+/**
+ * This class was created to extract out the logic from {@link Node#prepareForClose()} to facilitate testing.
+ * <p>
+ * Invokes hooks to prepare this node to be closed. This should be called when Elasticsearch receives a request to shut down
+ * gracefully from the underlying operating system, before system resources are closed.
+ * <p>
+ * Note that this class is part of infrastructure to react to signals from the operating system - most graceful shutdown
+ * logic should use Node Shutdown, see {@link org.elasticsearch.cluster.metadata.NodesShutdownMetadata}.
+ */
+public class ShutdownPrepareService {
+
+    private final Logger logger = LogManager.getLogger(ShutdownPrepareService.class);
+    private final Settings settings;
+    private final HttpServerTransport httpServerTransport;
+    private final TerminationHandler terminationHandler;
+    private volatile boolean hasBeenShutdown = false;
+
+    public ShutdownPrepareService(Settings settings, HttpServerTransport httpServerTransport, TerminationHandler terminationHandler) {
+        this.settings = settings;
+        this.httpServerTransport = httpServerTransport;
+        this.terminationHandler = terminationHandler;
+    }
+
+    public static final Setting<TimeValue> MAXIMUM_SHUTDOWN_TIMEOUT_SETTING = Setting.positiveTimeSetting(
+        "node.maximum_shutdown_grace_period",
+        TimeValue.ZERO,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<TimeValue> MAXIMUM_REINDEXING_TIMEOUT_SETTING = Setting.positiveTimeSetting(
+        "node.maximum_reindexing_grace_period",
+        TimeValue.timeValueSeconds(10),
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Invokes hooks to prepare this node to be closed. This should be called when Elasticsearch receives a request to shut down
+     * gracefully from the underlying operating system, before system resources are closed. This method will block
+     * until the node is ready to shut down.
+     * <p>
+     * Note that this class is part of infrastructure to react to signals from the operating system - most graceful shutdown
+     * logic should use Node Shutdown, see {@link org.elasticsearch.cluster.metadata.NodesShutdownMetadata}.
+     */
+    public void prepareForShutdown(TaskManager taskManager) {
+        assert hasBeenShutdown == false;
+        hasBeenShutdown = true;
+        final var maxTimeout = MAXIMUM_SHUTDOWN_TIMEOUT_SETTING.get(settings);
+        final var reindexTimeout = MAXIMUM_REINDEXING_TIMEOUT_SETTING.get(settings);
+
+        record Stopper(String name, SubscribableListener<Void> listener) {
+            boolean isIncomplete() {
+                return listener().isDone() == false;
+            }
+        }
+
+        final var stoppers = new ArrayList<Stopper>();
+        final var allStoppersFuture = new PlainActionFuture<Void>();
+        try (var listeners = new RefCountingListener(allStoppersFuture)) {
+            final BiConsumer<String, Runnable> stopperRunner = (name, action) -> {
+                final var stopper = new Stopper(name, new SubscribableListener<>());
+                stoppers.add(stopper);
+                stopper.listener().addListener(listeners.acquire());
+                new Thread(() -> {
+                    try {
+                        action.run();
+                    } catch (Exception ex) {
+                        logger.warn("unexpected exception in shutdown task [" + stopper.name() + "]", ex);
+                    } finally {
+                        stopper.listener().onResponse(null);
+                    }
+                }, stopper.name()).start();
+            };
+
+            stopperRunner.accept("http-server-transport-stop", httpServerTransport::close);
+            stopperRunner.accept("async-search-stop", () -> awaitSearchTasksComplete(maxTimeout, taskManager));
+            stopperRunner.accept("reindex-stop", () -> awaitReindexTasksComplete(reindexTimeout, taskManager));
+            if (terminationHandler != null) {
+                stopperRunner.accept("termination-handler-stop", terminationHandler::handleTermination);
+            }
+        }
+
+        final Supplier<String> incompleteStoppersDescriber = () -> stoppers.stream()
+            .filter(Stopper::isIncomplete)
+            .map(Stopper::name)
+            .collect(Collectors.joining(", ", "[", "]"));
+
+        try {
+            if (TimeValue.ZERO.equals(maxTimeout)) {
+                allStoppersFuture.get();
+            } else {
+                allStoppersFuture.get(maxTimeout.millis(), TimeUnit.MILLISECONDS);
+            }
+        } catch (ExecutionException e) {
+            assert false : e; // listeners are never completed exceptionally
+            logger.warn("failed during graceful shutdown tasks", e);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            logger.warn("interrupted while waiting for graceful shutdown tasks: " + incompleteStoppersDescriber.get(), e);
+        } catch (TimeoutException e) {
+            logger.warn("timed out while waiting for graceful shutdown tasks: " + incompleteStoppersDescriber.get());
+        }
+    }
+
+    private void awaitTasksComplete(TimeValue timeout, String taskName, TaskManager taskManager) {
+        long millisWaited = 0;
+        while (true) {
+            long tasksRemaining = taskManager.getTasks().values().stream().filter(task -> taskName.equals(task.getAction())).count();
+            if (tasksRemaining == 0) {
+                logger.debug("all " + taskName + " tasks complete");
+                return;
+            } else {
+                // Let the system work on those tasks for a while. We're on a dedicated thread to manage app shutdown, so we
+                // literally just want to wait and not take up resources on this thread for now. Poll period chosen to allow short
+                // response times, but checking the tasks list is relatively expensive, and we don't want to waste CPU time we could
+                // be spending on finishing those tasks.
+                final TimeValue pollPeriod = TimeValue.timeValueMillis(500);
+                millisWaited += pollPeriod.millis();
+                if (TimeValue.ZERO.equals(timeout) == false && millisWaited >= timeout.millis()) {
+                    logger.warn(
+                        format("timed out after waiting [%s] for [%d] " + taskName + " tasks to finish", timeout.toString(), tasksRemaining)
+                    );
+                    return;
+                }
+                logger.debug(format("waiting for [%s] " + taskName + " tasks to finish, next poll in [%s]", tasksRemaining, pollPeriod));
+                try {
+                    Thread.sleep(pollPeriod.millis());
+                } catch (InterruptedException ex) {
+                    logger.warn(
+                        format(
+                            "interrupted while waiting [%s] for [%d] " + taskName + " tasks to finish",
+                            timeout.toString(),
+                            tasksRemaining
+                        )
+                    );
+                    return;
+                }
+            }
+        }
+    }
+
+    private void awaitSearchTasksComplete(TimeValue asyncSearchTimeout, TaskManager taskManager) {
+        awaitTasksComplete(asyncSearchTimeout, TransportSearchAction.NAME, taskManager);
+    }
+
+    private void awaitReindexTasksComplete(TimeValue asyncReindexTimeout, TaskManager taskManager) {
+        awaitTasksComplete(asyncReindexTimeout, ReindexAction.NAME, taskManager);
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
index 55753f2827d2a..bde2d55ef2940 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
@@ -17,12 +17,11 @@
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -552,11 +551,8 @@ public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput in) throws IOE
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
-        return Iterators.concat(
-            Iterators.single((builder, params) -> builder.field("last_allocation_id", lastAllocationId)),
-            ChunkedToXContentHelper.array("tasks", tasks.values().iterator())
-        );
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContent.builder(params).field("last_allocation_id", lastAllocationId).array("tasks", tasks.values().iterator());
     }
 
     public static Builder builder() {
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index fc39d2d2d80a4..2b95fbc69199f 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -937,8 +937,7 @@ private static boolean indexSettingsMatchRepositoryMetadata(IndexMetadata indexM
     private static RepositoryConflictException newRepositoryConflictException(String repository, String reason) {
         return new RepositoryConflictException(
             repository,
-            "trying to modify or unregister repository that is currently used (" + reason + ')',
-            "trying to modify or unregister repository [" + repository + "] that is currently used (" + reason + ')'
+            "trying to modify or unregister repository that is currently used (" + reason + ')'
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java
index ee4ffa86a2475..15a6b0d3791d8 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -19,11 +20,8 @@
  * Repository conflict exception
  */
 public class RepositoryConflictException extends RepositoryException {
-    private final String backwardCompatibleMessage;
-
-    public RepositoryConflictException(String repository, String message, String backwardCompatibleMessage) {
+    public RepositoryConflictException(String repository, String message) {
         super(repository, message);
-        this.backwardCompatibleMessage = backwardCompatibleMessage;
     }
 
     @Override
@@ -31,18 +29,16 @@ public RestStatus status() {
         return RestStatus.CONFLICT;
     }
 
-    public String getBackwardCompatibleMessage() {
-        return backwardCompatibleMessage;
-    }
-
+    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // drop unneeded string from wire format
     public RepositoryConflictException(StreamInput in) throws IOException {
         super(in);
-        this.backwardCompatibleMessage = in.readString();
+        in.readString();
     }
 
     @Override
+    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // drop unneeded string from wire format
     protected void writeTo(StreamOutput out, Writer<Throwable> nestedExceptionsWriter) throws IOException {
         super.writeTo(out, nestedExceptionsWriter);
-        out.writeString(backwardCompatibleMessage);
+        out.writeString("");
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 5df7a8ea20f54..435bf71e3b2c9 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -4029,7 +4029,7 @@ protected void snapshotFile(SnapshotShardContext context, FileInfo fileInfo) thr
         final String file = fileInfo.physicalName();
         try (
             Releasable ignored = context.withCommitRef();
-            IndexInput indexInput = store.openVerifyingInput(file, IOContext.READ, fileInfo.metadata())
+            IndexInput indexInput = store.openVerifyingInput(file, IOContext.DEFAULT, fileInfo.metadata())
         ) {
             for (int i = 0; i < fileInfo.numberOfParts(); i++) {
                 final long partBytes = fileInfo.partBytes(i);
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java
index 1a58974985ba8..af0512b78cb7e 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ErrorState.java
@@ -15,9 +15,22 @@
 
 import static org.elasticsearch.ExceptionsHelper.stackTrace;
 
-record ErrorState(String namespace, Long version, List<String> errors, ReservedStateErrorMetadata.ErrorKind errorKind) {
-    ErrorState(String namespace, Long version, Exception e, ReservedStateErrorMetadata.ErrorKind errorKind) {
-        this(namespace, version, List.of(stackTrace(e)), errorKind);
+record ErrorState(
+    String namespace,
+    Long version,
+    ReservedStateVersionCheck versionCheck,
+    List<String> errors,
+    ReservedStateErrorMetadata.ErrorKind errorKind
+) {
+
+    ErrorState(
+        String namespace,
+        Long version,
+        ReservedStateVersionCheck versionCheck,
+        Exception e,
+        ReservedStateErrorMetadata.ErrorKind errorKind
+    ) {
+        this(namespace, version, versionCheck, List.of(stackTrace(e)), errorKind);
     }
 
     public String toString() {
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java
index c29f83c780d39..601fc3c86d98f 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -27,6 +28,8 @@
 import java.nio.file.Files;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.reservedstate.service.ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION;
+import static org.elasticsearch.reservedstate.service.ReservedStateVersionCheck.HIGHER_VERSION_ONLY;
 import static org.elasticsearch.xcontent.XContentType.JSON;
 
 /**
@@ -115,18 +118,41 @@ protected boolean shouldRefreshFileState(ClusterState clusterState) {
      */
     @Override
     protected void processFileChanges() throws ExecutionException, InterruptedException, IOException {
-        PlainActionFuture<Void> completion = new PlainActionFuture<>();
         logger.info("processing path [{}] for [{}]", watchedFile(), NAMESPACE);
+        processFileChanges(HIGHER_VERSION_ONLY);
+    }
+
+    /**
+     * Read settings and pass them to {@link ReservedClusterStateService} for application.
+     * Settings will be reprocessed even if the cluster-state version equals that found in the settings file.
+     */
+    @Override
+    protected void processFileOnServiceStart() throws IOException, ExecutionException, InterruptedException {
+        logger.info("processing path [{}] for [{}] on service start", watchedFile(), NAMESPACE);
+        processFileChanges(HIGHER_OR_SAME_VERSION);
+    }
+
+    private void processFileChanges(ReservedStateVersionCheck versionCheck) throws IOException, InterruptedException, ExecutionException {
+        PlainActionFuture<Void> completion = new PlainActionFuture<>();
         try (
             var fis = Files.newInputStream(watchedFile());
             var bis = new BufferedInputStream(fis);
             var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis)
         ) {
-            stateService.process(NAMESPACE, parser, (e) -> completeProcessing(e, completion));
+            stateService.process(NAMESPACE, parser, versionCheck, (e) -> completeProcessing(e, completion));
         }
         completion.get();
     }
 
+    @Override
+    protected void onProcessFileChangesException(Exception e) {
+        if (e instanceof ExecutionException && e.getCause() instanceof FailedToCommitClusterStateException f) {
+            logger.error("Unable to commit cluster state", e);
+        } else {
+            super.onProcessFileChangesException(e);
+        }
+    }
+
     @Override
     protected void processInitialFileMissing() throws ExecutionException, InterruptedException, IOException {
         PlainActionFuture<ActionResponse.Empty> completion = new PlainActionFuture<>();
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java
index 5571fcfb08544..499b5e6515a8c 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java
@@ -11,7 +11,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterState;
@@ -22,6 +21,7 @@
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -110,7 +110,13 @@ ReservedStateChunk parse(String namespace, XContentParser parser) {
         try {
             return stateChunkParser.apply(parser, null);
         } catch (Exception e) {
-            ErrorState errorState = new ErrorState(namespace, EMPTY_VERSION, e, ReservedStateErrorMetadata.ErrorKind.PARSING);
+            ErrorState errorState = new ErrorState(
+                namespace,
+                EMPTY_VERSION,
+                ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+                e,
+                ReservedStateErrorMetadata.ErrorKind.PARSING
+            );
             updateErrorState(errorState);
             logger.debug("error processing state change request for [{}] with the following errors [{}]", namespace, errorState);
 
@@ -123,16 +129,22 @@ ReservedStateChunk parse(String namespace, XContentParser parser) {
      *
      * @param namespace the namespace under which we'll store the reserved keys in the cluster state metadata
      * @param parser the XContentParser to process
+     * @param versionCheck determines if current and new versions of reserved state require processing or should be skipped
      * @param errorListener a consumer called with {@link IllegalStateException} if the content has errors and the
      *        cluster state cannot be correctly applied, null if successful or state couldn't be applied because of incompatible version.
      */
-    public void process(String namespace, XContentParser parser, Consumer<Exception> errorListener) {
+    public void process(
+        String namespace,
+        XContentParser parser,
+        ReservedStateVersionCheck versionCheck,
+        Consumer<Exception> errorListener
+    ) {
         ReservedStateChunk stateChunk;
 
         try {
             stateChunk = parse(namespace, parser);
         } catch (Exception e) {
-            ErrorState errorState = new ErrorState(namespace, EMPTY_VERSION, e, ReservedStateErrorMetadata.ErrorKind.PARSING);
+            ErrorState errorState = new ErrorState(namespace, EMPTY_VERSION, versionCheck, e, ReservedStateErrorMetadata.ErrorKind.PARSING);
             updateErrorState(errorState);
             logger.debug("error processing state change request for [{}] with the following errors [{}]", namespace, errorState);
 
@@ -142,17 +154,18 @@ public void process(String namespace, XContentParser parser, Consumer<Exception>
             return;
         }
 
-        process(namespace, stateChunk, errorListener);
+        process(namespace, stateChunk, versionCheck, errorListener);
     }
 
     public void initEmpty(String namespace, ActionListener<ActionResponse.Empty> listener) {
-        var missingVersion = new ReservedStateVersion(EMPTY_VERSION, Version.CURRENT);
+        var missingVersion = new ReservedStateVersion(EMPTY_VERSION, BuildVersion.current());
         var emptyState = new ReservedStateChunk(Map.of(), missingVersion);
         updateTaskQueue.submitTask(
             "empty initial cluster state [" + namespace + "]",
             new ReservedStateUpdateTask(
                 namespace,
                 emptyState,
+                ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
                 Map.of(),
                 List.of(),
                 // error state should not be possible since there is no metadata being parsed or processed
@@ -172,9 +185,14 @@ public void initEmpty(String namespace, ActionListener<ActionResponse.Empty> lis
      * @param errorListener a consumer called with {@link IllegalStateException} if the content has errors and the
      *        cluster state cannot be correctly applied, null if successful or the state failed to apply because of incompatible version.
      */
-    public void process(String namespace, ReservedStateChunk reservedStateChunk, Consumer<Exception> errorListener) {
+    public void process(
+        String namespace,
+        ReservedStateChunk reservedStateChunk,
+        ReservedStateVersionCheck versionCheck,
+        Consumer<Exception> errorListener
+    ) {
         Map<String, Object> reservedState = reservedStateChunk.state();
-        final ReservedStateVersion reservedStateVersion = reservedStateChunk.metadata();
+        ReservedStateVersion reservedStateVersion = reservedStateChunk.metadata();
 
         LinkedHashSet<String> orderedHandlers;
         try {
@@ -183,6 +201,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con
             ErrorState errorState = new ErrorState(
                 namespace,
                 reservedStateVersion.version(),
+                versionCheck,
                 e,
                 ReservedStateErrorMetadata.ErrorKind.PARSING
             );
@@ -201,7 +220,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con
 
         // We check if we should exit early on the state version from clusterService. The ReservedStateUpdateTask
         // will check again with the most current state version if this continues.
-        if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion) == false) {
+        if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion, versionCheck) == false) {
             errorListener.accept(null);
             return;
         }
@@ -209,7 +228,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con
         // We trial run all handler validations to ensure that we can process all of the cluster state error free.
         var trialRunErrors = trialRun(namespace, state, reservedStateChunk, orderedHandlers);
         // this is not using the modified trial state above, but that doesn't matter, we're just setting errors here
-        var error = checkAndReportError(namespace, trialRunErrors, reservedStateVersion);
+        var error = checkAndReportError(namespace, trialRunErrors, reservedStateVersion, versionCheck);
 
         if (error != null) {
             errorListener.accept(error);
@@ -220,6 +239,7 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con
             new ReservedStateUpdateTask(
                 namespace,
                 reservedStateChunk,
+                versionCheck,
                 handlers,
                 orderedHandlers,
                 ReservedClusterStateService.this::updateErrorState,
@@ -233,7 +253,7 @@ public void onResponse(ActionResponse.Empty empty) {
                     @Override
                     public void onFailure(Exception e) {
                         // Don't spam the logs on repeated errors
-                        if (isNewError(existingMetadata, reservedStateVersion.version())) {
+                        if (isNewError(existingMetadata, reservedStateVersion.version(), versionCheck)) {
                             logger.debug("Failed to apply reserved cluster state", e);
                             errorListener.accept(e);
                         } else {
@@ -247,7 +267,12 @@ public void onFailure(Exception e) {
     }
 
     // package private for testing
-    Exception checkAndReportError(String namespace, List<String> errors, ReservedStateVersion reservedStateVersion) {
+    Exception checkAndReportError(
+        String namespace,
+        List<String> errors,
+        ReservedStateVersion reservedStateVersion,
+        ReservedStateVersionCheck versionCheck
+    ) {
         // Any errors should be discovered through validation performed in the transform calls
         if (errors.isEmpty() == false) {
             logger.debug("Error processing state change request for [{}] with the following errors [{}]", namespace, errors);
@@ -255,6 +280,7 @@ Exception checkAndReportError(String namespace, List<String> errors, ReservedSta
             var errorState = new ErrorState(
                 namespace,
                 reservedStateVersion.version(),
+                versionCheck,
                 errors,
                 ReservedStateErrorMetadata.ErrorKind.VALIDATION
             );
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java
index 9296981e64d2d..e9fb736608d53 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java
@@ -51,10 +51,10 @@ ActionListener<ActionResponse.Empty> listener() {
     }
 
     // package private for testing
-    static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateVersion) {
+    static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateVersion, ReservedStateVersionCheck versionCheck) {
         return (existingMetadata == null
             || existingMetadata.errorMetadata() == null
-            || existingMetadata.errorMetadata().version() < newStateVersion
+            || versionCheck.test(existingMetadata.errorMetadata().version(), newStateVersion)
             || newStateVersion.equals(RESTORED_VERSION)
             || newStateVersion.equals(EMPTY_VERSION)
             || newStateVersion.equals(NO_VERSION));
@@ -63,7 +63,7 @@ static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateV
     static boolean checkErrorVersion(ClusterState currentState, ErrorState errorState) {
         ReservedStateMetadata existingMetadata = currentState.metadata().reservedStateMetadata().get(errorState.namespace());
         // check for noop here
-        if (isNewError(existingMetadata, errorState.version()) == false) {
+        if (isNewError(existingMetadata, errorState.version(), errorState.versionCheck()) == false) {
             logger.info(
                 () -> format(
                     "Not updating error state because version [%s] is less or equal to the last state error version [%s]",
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java
index 17d4de65506ff..90ae9923910d1 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java
@@ -11,7 +11,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterState;
@@ -47,6 +46,7 @@ public class ReservedStateUpdateTask implements ClusterStateTaskListener {
 
     private final String namespace;
     private final ReservedStateChunk stateChunk;
+    private final ReservedStateVersionCheck versionCheck;
     private final Map<String, ReservedClusterStateHandler<?>> handlers;
     private final Collection<String> orderedHandlers;
     private final Consumer<ErrorState> errorReporter;
@@ -55,6 +55,7 @@ public class ReservedStateUpdateTask implements ClusterStateTaskListener {
     public ReservedStateUpdateTask(
         String namespace,
         ReservedStateChunk stateChunk,
+        ReservedStateVersionCheck versionCheck,
         Map<String, ReservedClusterStateHandler<?>> handlers,
         Collection<String> orderedHandlers,
         Consumer<ErrorState> errorReporter,
@@ -62,6 +63,7 @@ public ReservedStateUpdateTask(
     ) {
         this.namespace = namespace;
         this.stateChunk = stateChunk;
+        this.versionCheck = versionCheck;
         this.handlers = handlers;
         this.orderedHandlers = orderedHandlers;
         this.errorReporter = errorReporter;
@@ -89,7 +91,7 @@ protected ClusterState execute(final ClusterState currentState) {
         Map<String, Object> reservedState = stateChunk.state();
         ReservedStateVersion reservedStateVersion = stateChunk.metadata();
 
-        if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion) == false) {
+        if (checkMetadataVersion(namespace, existingMetadata, reservedStateVersion, versionCheck) == false) {
             return currentState;
         }
 
@@ -110,7 +112,7 @@ protected ClusterState execute(final ClusterState currentState) {
             }
         }
 
-        checkAndThrowOnError(errors, reservedStateVersion);
+        checkAndThrowOnError(errors, reservedStateVersion, versionCheck);
 
         // Remove the last error if we had previously encountered any in prior processing of reserved state
         reservedMetadataBuilder.errorMetadata(null);
@@ -121,14 +123,15 @@ protected ClusterState execute(final ClusterState currentState) {
         return stateBuilder.metadata(metadataBuilder).build();
     }
 
-    private void checkAndThrowOnError(List<String> errors, ReservedStateVersion reservedStateVersion) {
+    private void checkAndThrowOnError(List<String> errors, ReservedStateVersion version, ReservedStateVersionCheck versionCheck) {
         // Any errors should be discovered through validation performed in the transform calls
         if (errors.isEmpty() == false) {
             logger.debug("Error processing state change request for [{}] with the following errors [{}]", namespace, errors);
 
             var errorState = new ErrorState(
                 namespace,
-                reservedStateVersion.version(),
+                version.version(),
+                versionCheck,
                 errors,
                 ReservedStateErrorMetadata.ErrorKind.VALIDATION
             );
@@ -155,48 +158,59 @@ static Set<String> keysForHandler(ReservedStateMetadata reservedStateMetadata, S
     static boolean checkMetadataVersion(
         String namespace,
         ReservedStateMetadata existingMetadata,
-        ReservedStateVersion reservedStateVersion
+        ReservedStateVersion reservedStateVersion,
+        ReservedStateVersionCheck versionCheck
     ) {
-        if (Version.CURRENT.before(reservedStateVersion.minCompatibleVersion())) {
+        if (reservedStateVersion.buildVersion().isFutureVersion()) {
             logger.warn(
                 () -> format(
                     "Reserved cluster state version [%s] for namespace [%s] is not compatible with this Elasticsearch node",
-                    reservedStateVersion.minCompatibleVersion(),
+                    reservedStateVersion.buildVersion(),
                     namespace
                 )
             );
             return false;
         }
 
-        if (reservedStateVersion.version().equals(ReservedStateMetadata.EMPTY_VERSION)) {
+        Long newVersion = reservedStateVersion.version();
+        if (newVersion.equals(ReservedStateMetadata.EMPTY_VERSION)) {
             return true;
         }
 
         // require a regular positive version, reject any special version
-        if (reservedStateVersion.version() <= 0L) {
+        if (newVersion <= 0L) {
             logger.warn(
                 () -> format(
                     "Not updating reserved cluster state for namespace [%s], because version [%s] is less or equal to 0",
                     namespace,
-                    reservedStateVersion.version()
+                    newVersion
                 )
             );
             return false;
         }
 
-        if (existingMetadata != null && existingMetadata.version() >= reservedStateVersion.version()) {
-            logger.warn(
-                () -> format(
-                    "Not updating reserved cluster state for namespace [%s], because version [%s] is less or equal"
-                        + " to the current metadata version [%s]",
-                    namespace,
-                    reservedStateVersion.version(),
-                    existingMetadata.version()
-                )
-            );
-            return false;
+        if (existingMetadata == null) {
+            return true;
+        }
+
+        Long currentVersion = existingMetadata.version();
+        if (versionCheck.test(currentVersion, newVersion)) {
+            return true;
         }
 
-        return true;
+        logger.warn(
+            () -> format(
+                "Not updating reserved cluster state for namespace [%s], because version [%s] is %s the current metadata version [%s]",
+                namespace,
+                newVersion,
+                switch (versionCheck) {
+                    case HIGHER_OR_SAME_VERSION -> "less than";
+                    case HIGHER_VERSION_ONLY -> "less than or equal to";
+                },
+                currentVersion
+            )
+        );
+        return false;
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java
index e2a21689b9815..116d470755e1c 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java
@@ -9,10 +9,10 @@
 
 package org.elasticsearch.reservedstate.service;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
@@ -23,7 +23,7 @@
  * File settings metadata class that holds information about
  * versioning and Elasticsearch version compatibility
  */
-public record ReservedStateVersion(Long version, Version compatibleWith) implements Writeable {
+public record ReservedStateVersion(Long version, BuildVersion buildVersion) implements Writeable {
 
     public static final ParseField VERSION = new ParseField("version");
     public static final ParseField COMPATIBILITY = new ParseField("compatibility");
@@ -32,7 +32,7 @@ public record ReservedStateVersion(Long version, Version compatibleWith) impleme
         "reserved_cluster_state_version_metadata",
         a -> {
             Long updateId = Long.parseLong((String) a[0]);
-            Version minCompatVersion = Version.fromString((String) a[1]);
+            BuildVersion minCompatVersion = BuildVersion.fromString((String) a[1]);
 
             return new ReservedStateVersion(updateId, minCompatVersion);
         }
@@ -47,17 +47,13 @@ public static ReservedStateVersion parse(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
-    public Version minCompatibleVersion() {
-        return compatibleWith;
-    }
-
     public static ReservedStateVersion readFrom(StreamInput input) throws IOException {
-        return new ReservedStateVersion(input.readLong(), Version.readVersion(input));
+        return new ReservedStateVersion(input.readLong(), BuildVersion.fromVersionId(input.readVInt()));
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(version());
-        Version.writeVersion(compatibleWith(), out);
+        out.writeVInt(buildVersion().id());
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersionCheck.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersionCheck.java
new file mode 100644
index 0000000000000..6907331edf1d6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersionCheck.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.reservedstate.service;
+
+import java.util.function.BiPredicate;
+
+/**
+ * Enum representing the logic for determining whether a reserved state should be processed
+ * based on the current and new versions.
+ */
+public enum ReservedStateVersionCheck implements BiPredicate<Long, Long> {
+    /**
+     * Returns {@code true} if the new version is higher than the current version.
+     * This is the default behavior when processing changes to file settings.
+     */
+    HIGHER_VERSION_ONLY {
+        @Override
+        public boolean test(Long currentVersion, Long newVersion) {
+            return currentVersion < newVersion;
+        }
+    },
+    /**
+     * Returns {@code true} if the new version is higher or equal to the current version.
+     * This allows re-processing of the same version.
+     * Used when processing file settings during service startup.
+     */
+    HIGHER_OR_SAME_VERSION {
+        @Override
+        public boolean test(Long currentVersion, Long newVersion) {
+            return currentVersion <= newVersion;
+        }
+    };
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index 2f7bb80a8d46a..0a99ee777bb76 100644
--- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -47,13 +47,6 @@
  */
 public abstract class BaseRestHandler implements RestHandler {
 
-    /**
-     * Parameter that controls whether certain REST apis should include type names in their requests or responses.
-     * Note: This parameter is only available through compatible rest api for {@link RestApiVersion#V_7}.
-     */
-    public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name";
-    public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false;
-
     public static final Setting<Boolean> MULTI_ALLOW_EXPLICIT_INDEX = Setting.boolSetting(
         "rest.action.multi.allow_explicit_index",
         true,
diff --git a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java
index b28d7c26b11bb..694af7e1606cb 100644
--- a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java
+++ b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasables;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Streams;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
@@ -129,9 +128,10 @@ public void write(byte[] b, int off, int len) throws IOException {
                 Streams.noCloseStream(out)
             );
 
-            private final Iterator<? extends ToXContent> serialization = builder.getRestApiVersion() == RestApiVersion.V_7
-                ? chunkedToXContent.toXContentChunkedV7(params)
-                : chunkedToXContent.toXContentChunked(params);
+            private final Iterator<? extends ToXContent> serialization = chunkedToXContent.toXContentChunked(
+                builder.getRestApiVersion(),
+                params
+            );
 
             private BytesStream target;
 
diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
index 0e3b8d37dd25c..cf66e402d3691 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -197,16 +197,6 @@ private RouteBuilder(Method method, String path) {
                 this.restApiVersion = RestApiVersion.current();
             }
 
-            /**
-            * @deprecated Use {@link #deprecatedForRemoval(String, RestApiVersion)} if the intent is deprecate the path and remove in the
-             * next major version. Use {@link #deprecateAndKeep(String)} if the intent is to deprecate the path but not remove it.
-             * This method will delegate to {@link #deprecatedForRemoval(String, RestApiVersion)}.
-             */
-            @Deprecated(since = "9.0.0", forRemoval = true)
-            public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFullySupportedVersion) {
-                return deprecatedForRemoval(deprecationMessage, lastFullySupportedVersion);
-            }
-
             /**
              * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage) for removal. Notes the last
              * major version in which the path is fully supported without compatibility headers. If this path is being replaced by another
diff --git a/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java b/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java
index 7f61d171fae33..db33673939ae9 100644
--- a/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java
+++ b/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Streams;
 import org.elasticsearch.transport.Transports;
 import org.elasticsearch.xcontent.ToXContent;
@@ -125,9 +124,7 @@ public void close() {
     }
 
     private Iterator<? extends ToXContent> getChunksIterator(StreamingFragment fragment) {
-        return xContentBuilder.getRestApiVersion() == RestApiVersion.V_7
-            ? fragment.fragment().toXContentChunkedV7(params)
-            : fragment.fragment().toXContentChunked(params);
+        return fragment.fragment().toXContentChunked(xContentBuilder.getRestApiVersion(), params);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java
index c8eb80cdcfdd7..e66b7d2b0b1a4 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -28,10 +27,6 @@
 public class RestAddVotingConfigExclusionAction extends BaseRestHandler {
     private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(30L);
 
-    private static final String DEPRECATION_MESSAGE = "POST /_cluster/voting_config_exclusions/{node_name} "
-        + "has been removed. You must use POST /_cluster/voting_config_exclusions?node_ids=... "
-        + "or POST /_cluster/voting_config_exclusions?node_names=... instead.";
-
     @Override
     public String getName() {
         return "add_voting_config_exclusions_action";
@@ -39,12 +34,7 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(POST, "/_cluster/voting_config_exclusions"),
-            Route.builder(POST, "/_cluster/voting_config_exclusions/{node_name}")
-                .deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, "/_cluster/voting_config_exclusions"));
     }
 
     @Override
@@ -66,10 +56,6 @@ static AddVotingConfigExclusionsRequest resolveVotingConfigExclusionsRequest(fin
         String nodeIds = null;
         String nodeNames = null;
 
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("node_name")) {
-            throw new IllegalArgumentException("[node_name] has been removed, you must set [node_names] or [node_ids]");
-        }
-
         if (request.hasParam("node_ids")) {
             nodeIds = request.param("node_ids");
         }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
index fada07d60b74e..970733b36f7f8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -95,20 +96,19 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             request.params().put("explain", Boolean.TRUE.toString());
         }
 
-        switch (request.getRestApiVersion()) {
-            case V_9 -> {
-                // always avoid returning the cluster state by forcing `?metric=none`; emit a warning if `?metric` is even present
-                if (request.hasParam("metric")) {
-                    deprecationLogger.critical(DeprecationCategory.API, "cluster-reroute-metric-param", METRIC_DEPRECATION_MESSAGE);
-                }
-                request.params().put("metric", "none");
+        if (request.getRestApiVersion().matches(RestApiVersion.onOrAfter(RestApiVersion.V_9))) {
+            // always avoid returning the cluster state by forcing `?metric=none`; emit a warning if `?metric` is even present
+            if (request.hasParam("metric")) {
+                deprecationLogger.critical(DeprecationCategory.API, "cluster-reroute-metric-param", METRIC_DEPRECATION_MESSAGE);
             }
-            case V_8, V_7 -> {
-                // by default, return everything but metadata
-                final String metric = request.param("metric");
-                if (metric == null) {
-                    request.params().put("metric", V8_DEFAULT_METRICS);
-                }
+            request.params().put("metric", "none");
+        } else {
+            assert request.getRestApiVersion().matches(RestApiVersion.equalTo(RestApiVersion.V_8));
+            @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // forbid this parameter in the v10 API
+            // by default, return everything but metadata
+            final String metric = request.param("metric");
+            if (metric == null) {
+                request.params().put("metric", V8_DEFAULT_METRICS);
             }
         }
 
@@ -117,6 +117,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             clusterRerouteRequest,
             new RestRefCountedChunkedToXContentListener<>(channel)
         );
+
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
index d2f5ca31ae3d1..c512db85d3023 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
@@ -10,10 +10,7 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.repositories.RepositoryConflictException;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -45,19 +42,11 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        String name = request.param("repository");
-        final var deleteRepositoryRequest = new DeleteRepositoryRequest(getMasterNodeTimeout(request), getAckTimeout(request), name);
-        return channel -> client.admin()
-            .cluster()
-            .deleteRepository(
-                deleteRepositoryRequest,
-                new RestToXContentListener<AcknowledgedResponse>(channel).delegateResponse((delegate, err) -> {
-                    if (request.getRestApiVersion().equals(RestApiVersion.V_7) && err instanceof RepositoryConflictException) {
-                        delegate.onFailure(new IllegalStateException(((RepositoryConflictException) err).getBackwardCompatibleMessage()));
-                    } else {
-                        delegate.onFailure(err);
-                    }
-                })
-            );
+        final var deleteRepositoryRequest = new DeleteRepositoryRequest(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            request.param("repository")
+        );
+        return channel -> client.admin().cluster().deleteRepository(deleteRepositoryRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
index e84ed3d59be1d..1302247a813f7 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -26,7 +25,6 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Locale;
 
 import static org.elasticsearch.rest.ChunkedRestResponseBodyPart.fromTextChunks;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -36,64 +34,9 @@
 @ServerlessScope(Scope.INTERNAL)
 public class RestNodesHotThreadsAction extends BaseRestHandler {
 
-    private static final String formatDeprecatedMessageWithoutNodeID = "[%s] is a deprecated endpoint. "
-        + "Please use [/_nodes/hot_threads] instead.";
-    private static final String formatDeprecatedMessageWithNodeID = "[%s] is a deprecated endpoint. "
-        + "Please use [/_nodes/{nodeId}/hot_threads] instead.";
-    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS = String.format(
-        Locale.ROOT,
-        formatDeprecatedMessageWithoutNodeID,
-        "/_cluster/nodes/hot_threads"
-    );
-    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS = String.format(
-        Locale.ROOT,
-        formatDeprecatedMessageWithNodeID,
-        "/_cluster/nodes/{nodeId}/hot_threads"
-    );
-    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS = String.format(
-        Locale.ROOT,
-        formatDeprecatedMessageWithoutNodeID,
-        "/_cluster/nodes/hotthreads"
-    );
-    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS = String.format(
-        Locale.ROOT,
-        formatDeprecatedMessageWithNodeID,
-        "/_cluster/nodes/{nodeId}/hotthreads"
-    );
-    private static final String DEPRECATED_MESSAGE_NODES_HOTTHREADS = String.format(
-        Locale.ROOT,
-        formatDeprecatedMessageWithoutNodeID,
-        "/_nodes/hotthreads"
-    );
-    private static final String DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS = String.format(
-        Locale.ROOT,
-        formatDeprecatedMessageWithNodeID,
-        "/_nodes/{nodeId}/hotthreads"
-    );
-
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(GET, "/_nodes/hot_threads"),
-            new Route(GET, "/_nodes/{nodeId}/hot_threads"),
-
-            Route.builder(GET, "/_cluster/nodes/hot_threads")
-                .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS, RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, "/_cluster/nodes/{nodeId}/hot_threads")
-                .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS, RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, "/_cluster/nodes/hotthreads")
-                .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS, RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, "/_cluster/nodes/{nodeId}/hotthreads")
-                .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS, RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, "/_nodes/hotthreads").deprecated(DEPRECATED_MESSAGE_NODES_HOTTHREADS, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_nodes/{nodeId}/hotthreads")
-                .deprecated(DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, "/_nodes/hot_threads"), new Route(GET, "/_nodes/{nodeId}/hot_threads"));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
index 933dd4d966fdf..c2848781bc7a7 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -72,11 +71,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("types")) {
-            deprecationLogger.compatibleCritical("nodes_stats_types", TYPES_DEPRECATION_MESSAGE);
-            request.param("types");
-        }
-
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         Set<String> metricNames = Strings.tokenizeByCommaToSet(request.param("metric", "_all"));
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
index 6f913932f4335..7419a589890e8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
@@ -10,10 +10,7 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.repositories.RepositoryConflictException;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -47,23 +44,15 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        String name = request.param("repository");
-        final var putRepositoryRequest = new PutRepositoryRequest(getMasterNodeTimeout(request), getAckTimeout(request), name);
+        final var putRepositoryRequest = new PutRepositoryRequest(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            request.param("repository")
+        );
         try (XContentParser parser = request.contentParser()) {
             putRepositoryRequest.source(parser.mapOrdered());
         }
         putRepositoryRequest.verify(request.paramAsBoolean("verify", true));
-        return channel -> client.admin()
-            .cluster()
-            .putRepository(
-                putRepositoryRequest,
-                new RestToXContentListener<AcknowledgedResponse>(channel).delegateResponse((delegate, err) -> {
-                    if (request.getRestApiVersion().equals(RestApiVersion.V_7) && err instanceof RepositoryConflictException) {
-                        delegate.onFailure(new IllegalStateException(((RepositoryConflictException) err).getBackwardCompatibleMessage()));
-                    } else {
-                        delegate.onFailure(err);
-                    }
-                })
-            );
+        return channel -> client.admin().cluster().putRepository(putRepositoryRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java
index ec8bb6285bdd4..b8e1fa0c836a3 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java
@@ -12,13 +12,11 @@
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.cluster.metadata.DesiredNode;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -67,16 +65,6 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             );
         }
 
-        if (clusterSupportsFeature.test(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED)) {
-            if (updateDesiredNodesRequest.getNodes().stream().anyMatch(DesiredNode::hasVersion)) {
-                deprecationLogger.compatibleCritical("desired_nodes_version", VERSION_DEPRECATION_MESSAGE);
-            }
-        } else {
-            if (updateDesiredNodesRequest.getNodes().stream().anyMatch(n -> n.hasVersion() == false)) {
-                throw new XContentParseException("[node_version] field is required and must have a valid value");
-            }
-        }
-
         return restChannel -> client.execute(
             UpdateDesiredNodesAction.INSTANCE,
             updateDesiredNodesRequest,
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
index 899486399af6b..900a352d42f30 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
@@ -21,5 +21,10 @@ public class CreateIndexCapabilities {
      */
     private static final String LOGSDB_INDEX_MODE_CAPABILITY = "logsdb_index_mode";
 
-    public static Set<String> CAPABILITIES = Set.of(LOGSDB_INDEX_MODE_CAPABILITY);
+    /**
+     * Support lookup index mode
+     */
+    private static final String LOOKUP_INDEX_MODE_CAPABILITY = "lookup_index_mode";
+
+    public static Set<String> CAPABILITIES = Set.of(LOGSDB_INDEX_MODE_CAPABILITY, LOOKUP_INDEX_MODE_CAPABILITY);
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
index 8e1a122f98a3a..e30d2f8d5c733 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
@@ -12,11 +12,8 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -25,7 +22,6 @@
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -39,10 +35,6 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestCreateIndexAction extends BaseRestHandler {
 
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestCreateIndexAction.class);
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in create "
-        + "index requests is deprecated. The parameter will be removed in the next major version.";
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(PUT, "/{index}"));
@@ -55,61 +47,10 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        CreateIndexRequest createIndexRequest;
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            createIndexRequest = prepareRequestV7(request);
-        } else {
-            createIndexRequest = prepareRequest(request);
-        }
+        final var createIndexRequest = prepareRequest(request);
         return channel -> client.admin().indices().create(createIndexRequest, new RestToXContentListener<>(channel));
     }
 
-    // default scope for testing types in mapping
-    static CreateIndexRequest prepareRequestV7(RestRequest request) {
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index"));
-        if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) {
-            request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc
-            deprecationLogger.compatibleCritical("create_index_with_types", TYPES_DEPRECATION_MESSAGE);
-        }
-
-        if (request.hasContent()) {
-            Map<String, Object> sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2();
-
-            sourceAsMap = prepareMappingsV7(sourceAsMap, request);
-
-            createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE);
-        }
-
-        createIndexRequest.ackTimeout(getAckTimeout(request));
-        createIndexRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-        createIndexRequest.waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards")));
-        return createIndexRequest;
-    }
-
-    static Map<String, Object> prepareMappingsV7(Map<String, Object> source, RestRequest request) {
-        final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, false);
-
-        @SuppressWarnings("unchecked")
-        Map<String, Object> mappings = (Map<String, Object>) source.get("mappings");
-
-        if (includeTypeName && mappings != null && mappings.size() == 1) {
-            Map<String, Object> newSource = new HashMap<>();
-            newSource.putAll(source); // mappings will be overridden. Aliases, settings stay the same
-            String typeName = mappings.keySet().iterator().next();
-            if (Strings.hasText(typeName) == false) {
-                throw new IllegalArgumentException("name cannot be empty string");
-            }
-            @SuppressWarnings("unchecked")
-            Map<String, Object> typedMappings = (Map<String, Object>) mappings.get(typeName);
-
-            // no matter what the type was, replace it with _doc, because the internal representation still uses single type `_doc`.
-            newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings));
-            return newSource;
-        } else {
-            return prepareMappings(source);
-        }
-    }
-
     static CreateIndexRequest prepareRequest(RestRequest request) {
         CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index"));
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
index 79e5c574d93bf..dfe501f29ce2e 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -52,7 +52,7 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetAliasesAction extends BaseRestHandler {
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) // reject the deprecated ?local parameter
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) // remove the BWC support for the deprecated ?local parameter
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RestGetAliasesAction.class);
 
     @Override
@@ -199,6 +199,7 @@ static RestResponse buildRestResponse(
     }
 
     @Override
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) // remove the BWC support for the deprecated ?local parameter
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         // The TransportGetAliasesAction was improved do the same post processing as is happening here.
         // We can't remove this logic yet to support mixed clusters. We should be able to remove this logic here
@@ -211,10 +212,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         getAliasesRequest.indices(indices);
         getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
 
-        if (request.hasParam("local")) {
-            // consume this param just for validation
-            final var localParam = request.paramAsBoolean("local", false);
-            if (request.getRestApiVersion() != RestApiVersion.V_7) {
+        if (request.getRestApiVersion() == RestApiVersion.V_8) {
+            if (request.hasParam("local")) {
+                // consume this param just for validation when in BWC mode for V_8
+                final var localParam = request.paramAsBoolean("local", false);
                 DEPRECATION_LOGGER.critical(
                     DeprecationCategory.API,
                     "get-aliases-local",
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
index 66465676ba53c..5f648ca8e77e5 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -45,13 +44,7 @@ public class RestGetFieldMappingAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(GET, "/_mapping/field/{fields}"),
-            new Route(GET, "/{index}/_mapping/field/{fields}"),
-            Route.builder(GET, "/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/{type}/_mapping/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_mapping/field/{fields}"), new Route(GET, "/{index}/_mapping/field/{fields}"));
     }
 
     @Override
@@ -64,25 +57,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final String[] fields = Strings.splitStringByCommaToArray(request.param("fields"));
 
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) {
-                deprecationLogger.compatibleCritical("get_field_mapping_with_types", INCLUDE_TYPE_DEPRECATION_MESSAGE);
-            }
-            boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY);
-            final String[] types = request.paramAsStringArrayOrEmptyIfAll("type");
-            if (includeTypeName == false && types.length > 0) {
-                throw new IllegalArgumentException("Types cannot be specified unless include_type_name is set to true.");
-            }
-
-            if (request.hasParam("local")) {
-                request.param("local");
-                deprecationLogger.compatibleCritical(
-                    "get_field_mapping_local",
-                    "Use [local] in get field mapping requests is deprecated. The parameter will be removed in the next major version"
-                );
-            }
-        }
-
         GetFieldMappingsRequest getMappingsRequest = new GetFieldMappingsRequest();
         getMappingsRequest.indices(indices).fields(fields).includeDefaults(request.paramAsBoolean("include_defaults", false));
         getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions()));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
index 25f471183d805..b0b879a6d787c 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
@@ -12,9 +12,7 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -23,7 +21,6 @@
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.common.util.set.Sets.addToCopy;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
@@ -34,13 +31,6 @@
  * The REST handler for get template and head template APIs.
  */
 public class RestGetIndexTemplateAction extends BaseRestHandler {
-
-    private static final Set<String> COMPATIBLE_RESPONSE_PARAMS = addToCopy(Settings.FORMAT_PARAMS, INCLUDE_TYPE_NAME_PARAMETER);
-
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndexTemplateAction.class);
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get "
-        + "index template requests is deprecated. The parameter will be removed in the next major version.";
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/_template"), new Route(GET, "/_template/{name}"), new Route(HEAD, "/_template/{name}"));
@@ -53,9 +43,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) {
-            deprecationLogger.compatibleCritical("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE);
-        }
         final String[] names = Strings.splitStringByCommaToArray(request.param("name"));
 
         final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names);
@@ -75,13 +62,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
     protected Set<String> responseParams() {
         return Settings.FORMAT_PARAMS;
     }
-
-    @Override
-    protected Set<String> responseParams(RestApiVersion restApiVersion) {
-        if (restApiVersion == RestApiVersion.V_7) {
-            return COMPATIBLE_RESPONSE_PARAMS;
-        } else {
-            return responseParams();
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
index 3d78fa538bf04..9ca890eaff65b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -27,7 +26,6 @@
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.common.util.set.Sets.addToCopy;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
@@ -41,8 +39,6 @@ public class RestGetIndicesAction extends BaseRestHandler {
     public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests"
         + " is deprecated. The parameter will be removed in the next major version.";
 
-    private static final Set<String> COMPATIBLE_RESPONSE_PARAMS = addToCopy(Settings.FORMAT_PARAMS, INCLUDE_TYPE_NAME_PARAMETER);
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/{index}"), new Route(HEAD, "/{index}"));
@@ -55,13 +51,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        // starting with 7.0 we don't include types by default in the response to GET requests
-        if (request.getRestApiVersion() == RestApiVersion.V_7
-            && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)
-            && request.method().equals(GET)) {
-            deprecationLogger.compatibleCritical("get_indices_with_types", TYPES_DEPRECATION_MESSAGE);
-        }
-
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final GetIndexRequest getIndexRequest = new GetIndexRequest();
         getIndexRequest.indices(indices);
@@ -85,13 +74,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
     protected Set<String> responseParams() {
         return Settings.FORMAT_PARAMS;
     }
-
-    @Override
-    protected Set<String> responseParams(RestApiVersion restApiVersion) {
-        if (restApiVersion == RestApiVersion.V_7) {
-            return COMPATIBLE_RESPONSE_PARAMS;
-        } else {
-            return responseParams();
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
index 2aa230a13c2a9..242bcd399413b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -28,7 +27,6 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -46,13 +44,8 @@ public List<Route> routes() {
         return List.of(
             new Route(GET, "/_mapping"),
             new Route(GET, "/_mappings"),
-            Route.builder(GET, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
             new Route(GET, "/{index}/_mapping"),
-            new Route(GET, "/{index}/_mappings"),
-            Route.builder(GET, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(HEAD, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(GET, "/{index}/_mappings")
         );
     }
 
@@ -63,26 +56,7 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) {
-                request.param(INCLUDE_TYPE_NAME_PARAMETER);
-                deprecationLogger.compatibleCritical("get_mapping_with_types", INCLUDE_TYPE_DEPRECATION_MSG);
-            }
-            final String[] types = request.paramAsStringArrayOrEmptyIfAll("type");
-            if (request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY) == false && types.length > 0) {
-                throw new IllegalArgumentException(
-                    "Types cannot be provided in get mapping requests, unless include_type_name is set to true."
-                );
-            }
-            if (request.method().equals(HEAD)) {
-                deprecationLogger.compatibleCritical(
-                    "get_mapping_types_removal",
-                    "Type exists requests are deprecated, as types have been deprecated."
-                );
-            }
-        }
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-
         final GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
         getMappingsRequest.indices(indices);
         getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions()));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
index dcd4cda21f969..b5050eb8007a8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -76,11 +75,6 @@ public boolean allowSystemIndexAccessByDefault() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("types")) {
-            deprecationLogger.compatibleCritical("indices_stats_types", TYPES_DEPRECATION_MESSAGE);
-            request.param("types");
-        }
-
         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
         boolean forbidClosedIndices = request.paramAsBoolean("forbid_closed_indices", true);
         IndicesOptions defaultIndicesOption = forbidClosedIndices
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
index f70d9351e69c9..592679a2a02a0 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
@@ -12,30 +12,23 @@
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Map;
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction.prepareMappings;
 
 public class RestPutIndexTemplateAction extends BaseRestHandler {
 
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestPutIndexTemplateAction.class);
     public static final String DEPRECATION_WARNING = "Legacy index templates are deprecated in favor of composable templates.";
-    private static final RestApiVersion DEPRECATION_VERSION = RestApiVersion.V_8;
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]"
-        + " Specifying include_type_name in put index template requests is deprecated."
-        + " The parameter will be removed in the next major version.";
 
     @Override
     public List<Route> routes() {
@@ -53,42 +46,12 @@ public String getName() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(request.param("name"));
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("template")) {
-            deprecationLogger.compatibleCritical(
-                "template_parameter_deprecation",
-                "Deprecated parameter [template] used, replaced by [index_patterns]"
-            );
-            putRequest.patterns(List.of(request.param("template")));
-        } else {
-            putRequest.patterns(asList(request.paramAsStringArray("index_patterns", Strings.EMPTY_ARRAY)));
-        }
+        putRequest.patterns(asList(request.paramAsStringArray("index_patterns", Strings.EMPTY_ARRAY)));
         putRequest.order(request.paramAsInt("order", putRequest.order()));
         putRequest.masterNodeTimeout(getMasterNodeTimeout(request));
         putRequest.create(request.paramAsBoolean("create", false));
         putRequest.cause(request.param("cause", ""));
-
-        Map<String, Object> sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2();
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) {
-                deprecationLogger.compatibleCritical("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE);
-            }
-            boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY);
-            if (includeTypeName) {
-                sourceAsMap = RestCreateIndexAction.prepareMappingsV7(sourceAsMap, request);
-            } else {
-                sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap);
-            }
-        } else {
-            sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap);
-        }
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && sourceAsMap.containsKey("template")) {
-            deprecationLogger.compatibleCritical(
-                "template_field_deprecation",
-                "Deprecated field [template] used, replaced by [index_patterns]"
-            );
-            putRequest.patterns(List.of((String) sourceAsMap.remove("template")));
-        }
-        putRequest.source(sourceAsMap);
+        putRequest.source(prepareMappings(XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2()));
 
         return channel -> client.admin().indices().putTemplate(putRequest, new RestToXContentListener<>(channel));
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
index ae5e46866f5a6..ee11f7f520463 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
@@ -15,8 +15,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -27,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
 import static org.elasticsearch.index.mapper.MapperService.isMappingSourceTyped;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -45,19 +44,7 @@ public List<Route> routes() {
             new Route(POST, "/{index}/_mapping/"),
             new Route(PUT, "/{index}/_mapping/"),
             new Route(POST, "/{index}/_mappings/"),
-            new Route(PUT, "/{index}/_mappings/"),
-            Route.builder(POST, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_mappings").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/{index}/{type}/_mappings").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(PUT, "/{index}/_mappings/")
         );
     }
 
@@ -72,27 +59,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         PutMappingRequest putMappingRequest = new PutMappingRequest(indices);
 
         Map<String, Object> sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2();
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY);
-            if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) {
-                deprecationLogger.compatibleCritical("put_mapping_with_types", TYPES_DEPRECATION_MESSAGE);
-            }
-            final String type = request.param("type");
-            if (includeTypeName == false && (type != null || isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap))) {
-                throw new IllegalArgumentException(
-                    "Types cannot be provided in put mapping requests, unless the include_type_name parameter is set to true."
-                );
-            }
-
-            Map<String, Object> mappingSource = prepareV7Mappings(includeTypeName, sourceAsMap);
-            putMappingRequest.source(mappingSource);
-        } else {
-            if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) {
-                throw new IllegalArgumentException("Types cannot be provided in put mapping requests");
-            }
-            putMappingRequest.source(sourceAsMap);
+        if (isMappingSourceTyped(SINGLE_MAPPING_NAME, sourceAsMap)) {
+            throw new IllegalArgumentException("Types cannot be provided in put mapping requests");
         }
-
+        putMappingRequest.source(sourceAsMap);
         putMappingRequest.ackTimeout(getAckTimeout(request));
         putMappingRequest.masterNodeTimeout(getMasterNodeTimeout(request));
         putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions()));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java
index c6afc8e041d01..ee1710f39ce41 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java
@@ -14,8 +14,6 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.Booleans;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -40,15 +38,6 @@ public abstract class RestResizeHandler extends BaseRestHandler {
 
     @Override
     public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("copy_settings")) {
-            deprecationLogger.compatibleCritical("copy_settings", "parameter [copy_settings] is deprecated and will be removed in 8.0.0");
-
-            final String rawCopySettings = request.param("copy_settings");
-            final boolean copySettings = Booleans.parseBoolean(rawCopySettings);
-            if (copySettings == false) {
-                throw new IllegalArgumentException("parameter [copy_settings] can not be explicitly set to [false]");
-            }
-        }
         final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index"));
         resizeRequest.setResizeType(getResizeType());
         request.applyContentParser(resizeRequest::fromXContent);
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index c4df68098190f..776302296b1a2 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -59,9 +58,8 @@ public Set<String> supportedCapabilities() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        final boolean includeTypeName = includeTypeName(request);
         RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index"));
-        request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser));
+        request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(parser));
         rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false));
         rolloverIndexRequest.lazy(request.paramAsBoolean("lazy", false));
         rolloverIndexRequest.ackTimeout(getAckTimeout(request));
@@ -71,7 +69,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             if (failureStore) {
                 rolloverIndexRequest.setIndicesOptions(
                     IndicesOptions.builder(rolloverIndexRequest.indicesOptions())
-                        .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true))
+                        .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
                         .build()
                 );
             }
@@ -83,14 +81,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             .rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel));
     }
 
-    private static boolean includeTypeName(RestRequest request) {
-        boolean includeTypeName = false;
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) {
-                deprecationLogger.compatibleCritical("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE);
-            }
-            includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY);
-        }
-        return includeTypeName;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
deleted file mode 100644
index d0a4160647f1b..0000000000000
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.rest.action.admin.indices;
-
-import org.elasticsearch.action.admin.indices.flush.FlushRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-
-public class RestSyncedFlushAction extends BaseRestHandler {
-
-    private static final String DEPRECATION_MESSAGE =
-        "Synced flush is deprecated and will be removed in 8.0. Use flush at /_flush or /{index}/_flush instead.";
-
-    @Override
-    public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
-    }
-
-    @Override
-    public String getName() {
-        return "synced_flush_action";
-    }
-
-    @Override
-    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        final FlushRequest flushRequest = new FlushRequest(Strings.splitStringByCommaToArray(request.param("index")));
-        flushRequest.indicesOptions(IndicesOptions.fromRequest(request, flushRequest.indicesOptions()));
-        return channel -> client.admin().indices().flush(flushRequest, new SimulateSyncedFlushResponseListener(channel));
-    }
-
-    static final class SimulateSyncedFlushResponseListener extends RestBuilderListener<BroadcastResponse> {
-
-        SimulateSyncedFlushResponseListener(RestChannel channel) {
-            super(channel);
-        }
-
-        @Override
-        public RestResponse buildResponse(BroadcastResponse flushResponse, XContentBuilder builder) throws Exception {
-            builder.startObject();
-            buildSyncedFlushResponse(builder, flushResponse);
-            builder.endObject();
-            final RestStatus restStatus = flushResponse.getFailedShards() == 0 ? RestStatus.OK : RestStatus.CONFLICT;
-            return new RestResponse(restStatus, builder);
-        }
-
-        private static void buildSyncedFlushResponse(XContentBuilder builder, BroadcastResponse flushResponse) throws IOException {
-            builder.startObject("_shards");
-            builder.field("total", flushResponse.getTotalShards());
-            builder.field("successful", flushResponse.getSuccessfulShards());
-            builder.field("failed", flushResponse.getFailedShards());
-            // can't serialize the detail of each index as we don't have the shard count per index.
-            builder.endObject();
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeActionDeprecated.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeActionDeprecated.java
deleted file mode 100644
index 148924f522e5e..0000000000000
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeActionDeprecated.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.rest.action.admin.indices;
-
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-
-public class RestUpgradeActionDeprecated extends BaseRestHandler {
-    public static final String UPGRADE_API_DEPRECATION_MESSAGE =
-        "The _upgrade API is no longer useful and will be removed. Instead, see _reindex API.";
-
-    @Override
-    public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
-    }
-
-    @Override
-    public String getName() {
-        return "upgrade_action";
-    }
-
-    @Override
-    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        request.param("index");
-        final UpgradeActionDeprecatedException exception = new UpgradeActionDeprecatedException(request);
-        return channel -> channel.sendResponse(new RestResponse(channel, exception));
-    }
-
-    public static class UpgradeActionDeprecatedException extends IllegalArgumentException {
-        private final String path;
-        private final RestRequest.Method method;
-
-        public UpgradeActionDeprecatedException(RestRequest restRequest) {
-            this.path = restRequest.path();
-            this.method = restRequest.method();
-        }
-
-        @Override
-        public final String getMessage() {
-            return String.format(Locale.ROOT, "Upgrade action %s %s was removed, use _reindex API instead", method, path);
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
index e5b88d101f11a..8bc7e9aa76551 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -46,9 +45,7 @@ public List<Route> routes() {
             new Route(GET, "/_validate/query"),
             new Route(POST, "/_validate/query"),
             new Route(GET, "/{index}/_validate/query"),
-            new Route(POST, "/{index}/_validate/query"),
-            Route.builder(GET, "/{index}/{type}/_validate/query").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_validate/query").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_validate/query")
         );
     }
 
@@ -59,11 +56,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            deprecationLogger.compatibleCritical("validate_query_with_types", TYPES_DEPRECATION_MESSAGE);
-            request.param("type");
-        }
-
         ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest(Strings.splitStringByCommaToArray(request.param("index")));
         validateQueryRequest.indicesOptions(IndicesOptions.fromRequest(request, validateQueryRequest.indicesOptions()));
         validateQueryRequest.explain(request.paramAsBoolean("explain", false));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index 0957821ef5531..6aa0b1c865682 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -15,9 +15,6 @@
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.logging.DeprecationCategory;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.Scope;
@@ -33,8 +30,6 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestAliasAction extends AbstractCatAction {
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RestAliasAction.class);
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/_cat/aliases"), new Route(GET, "/_cat/aliases/{alias}"));
@@ -56,20 +51,6 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, final Node
             ? new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias")))
             : new GetAliasesRequest();
         getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
-
-        if (request.hasParam("local")) {
-            // consume this param just for validation
-            final var localParam = request.paramAsBoolean("local", false);
-            if (request.getRestApiVersion() != RestApiVersion.V_7) {
-                DEPRECATION_LOGGER.critical(
-                    DeprecationCategory.API,
-                    "cat-aliases-local",
-                    "the [?local={}] query parameter to cat-aliases requests has no effect and will be removed in a future version",
-                    localParam
-                );
-            }
-        }
-
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin()
             .indices()
             .getAliases(getAliasesRequest, new RestResponseListener<>(channel) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
index a457ebb67fd47..09f31abb58eb3 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
@@ -72,7 +72,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
         return channel -> client.search(countRequest, new RestResponseListener<SearchResponse>(channel) {
             @Override
             public RestResponse buildResponse(SearchResponse countResponse) throws Exception {
-                assert countResponse.getHits().getTotalHits().relation == TotalHits.Relation.EQUAL_TO;
+                assert countResponse.getHits().getTotalHits().relation() == TotalHits.Relation.EQUAL_TO;
                 return RestTable.buildResponse(buildTable(request, countResponse), channel);
             }
         });
@@ -90,7 +90,7 @@ protected Table getTableWithHeader(final RestRequest request) {
     private Table buildTable(RestRequest request, SearchResponse response) {
         Table table = getTableWithHeader(request);
         table.startRow();
-        table.addCell(response.getHits().getTotalHits().value);
+        table.addCell(response.getHits().getTotalHits().value());
         table.endRow();
 
         return table;
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index f97ea1db7a036..a968ea4520f40 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.rest.RestRequest;
@@ -171,11 +170,7 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas");
         table.addCell("pri.store.size", "text-align:right;desc:store size of primaries");
 
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            table.addCell("dataset.size", "default:false;text-align:right;desc:total size of dataset");
-        } else {
-            table.addCell("dataset.size", "text-align:right;desc:total size of dataset");
-        }
+        table.addCell("dataset.size", "text-align:right;desc:total size of dataset");
 
         table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion");
         table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion");
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 72f786004e0bd..3bee9c5f3b04c 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.bulk.stats.BulkStats;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
@@ -119,12 +118,7 @@ protected Table getTableWithHeader(final RestRequest request) {
             .addCell("state", "default:true;alias:st;desc:shard state")
             .addCell("docs", "alias:d,dc;text-align:right;desc:number of docs in shard")
             .addCell("store", "alias:sto;text-align:right;desc:store size of shard (how much disk it uses)")
-            .addCell(
-                "dataset",
-                request.getRestApiVersion() == RestApiVersion.V_7
-                    ? "default:false;text-align:right;desc:total size of dataset"
-                    : "text-align:right;desc:total size of dataset"
-            )
+            .addCell("dataset", "text-align:right;desc:total size of dataset")
             .addCell("ip", "default:true;desc:ip of node where it lives")
             .addCell("id", "default:false;desc:unique id of node where it lives")
             .addCell("node", "default:true;alias:n;desc:name of node where it lives");
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
index 33ee87c0c0b5a..1e80e6de60d65 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
@@ -79,9 +78,7 @@ public List<Route> routes() {
             new Route(POST, "/_bulk"),
             new Route(PUT, "/_bulk"),
             new Route(POST, "/{index}/_bulk"),
-            new Route(PUT, "/{index}/_bulk"),
-            Route.builder(POST, "/{index}/{type}/_bulk").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/{index}/{type}/_bulk").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(PUT, "/{index}/_bulk")
         );
     }
 
@@ -93,9 +90,6 @@ public String getName() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         if (request.isStreamedContent() == false) {
-            if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-                request.param("type");
-            }
             BulkRequest bulkRequest = new BulkRequest();
             String defaultIndex = request.param("index");
             String defaultRouting = request.param("routing");
@@ -110,26 +104,26 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             boolean defaultRequireDataStream = request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, false);
             bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
             bulkRequest.setRefreshPolicy(request.param("refresh"));
-            bulkRequest.add(
-                request.requiredContent(),
-                defaultIndex,
-                defaultRouting,
-                defaultFetchSourceContext,
-                defaultPipeline,
-                defaultRequireAlias,
-                defaultRequireDataStream,
-                defaultListExecutedPipelines,
-                allowExplicitIndex,
-                request.getXContentType(),
-                request.getRestApiVersion()
-            );
+            try {
+                bulkRequest.add(
+                    request.requiredContent(),
+                    defaultIndex,
+                    defaultRouting,
+                    defaultFetchSourceContext,
+                    defaultPipeline,
+                    defaultRequireAlias,
+                    defaultRequireDataStream,
+                    defaultListExecutedPipelines,
+                    allowExplicitIndex,
+                    request.getXContentType(),
+                    request.getRestApiVersion()
+                );
+            } catch (Exception e) {
+                return channel -> new RestToXContentListener<>(channel).onFailure(parseFailureException(e));
+            }
 
             return channel -> client.bulk(bulkRequest, new RestRefCountedChunkedToXContentListener<>(channel));
         } else {
-            if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-                request.param("type");
-            }
-
             String waitForActiveShards = request.param("wait_for_active_shards");
             TimeValue timeout = request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT);
             String refresh = request.param("refresh");
@@ -137,6 +131,15 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         }
     }
 
+    private static Exception parseFailureException(Exception e) {
+        if (e instanceof IllegalArgumentException) {
+            return e;
+        } else {
+            // TODO: Maybe improve in follow-up to be XContentParseException and include line number and column
+            return new ElasticsearchParseException("could not parse bulk request body", e);
+        }
+    }
+
     static class ChunkHandler implements BaseRestHandler.RequestBodyChunkConsumer {
 
         private final boolean allowExplicitIndex;
@@ -229,9 +232,7 @@ public void handleChunk(RestChannel channel, ReleasableBytesReference chunk, boo
 
                 } catch (Exception e) {
                     shortCircuit();
-                    new RestToXContentListener<>(channel).onFailure(
-                        new ElasticsearchParseException("could not parse bulk request body", e)
-                    );
+                    new RestToXContentListener<>(channel).onFailure(parseFailureException(e));
                     return;
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
index f70335a7d39be..3c795c635056d 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -29,15 +28,10 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteAction extends BaseRestHandler {
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
-        + "document index requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(DELETE, "/{index}/_doc/{id}"),
-            Route.builder(DELETE, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(DELETE, "/{index}/_doc/{id}"));
     }
 
     @Override
@@ -47,9 +41,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type");
-        }
         DeleteRequest deleteRequest = new DeleteRequest(request.param("index"), request.param("id"));
         deleteRequest.routing(request.param("routing"));
         deleteRequest.timeout(request.paramAsTime("timeout", DeleteRequest.DEFAULT_TIMEOUT));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index 9018e3e0806d4..4e234c4cbcc3d 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -32,9 +31,6 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetAction extends BaseRestHandler {
-    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
-        + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.";
-
     @Override
     public String getName() {
         return "document_get_action";
@@ -42,22 +38,12 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(GET, "/{index}/_doc/{id}"),
-            new Route(HEAD, "/{index}/_doc/{id}"),
-            Route.builder(GET, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(HEAD, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/{index}/_doc/{id}"), new Route(HEAD, "/{index}/_doc/{id}"));
     }
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            request.param("type"); // consume and ignore the type
-        }
-
         GetRequest getRequest = new GetRequest(request.param("index"), request.param("id"));
-
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
         getRequest.routing(request.param("routing"));
         getRequest.preference(request.param("preference"));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
index a71bf9c305185..a09fcbd0c5273 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -47,12 +46,7 @@ public class RestGetSourceAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(GET, "/{index}/_source/{id}"),
-            new Route(HEAD, "/{index}/_source/{id}"),
-            Route.builder(GET, "/{index}/{type}/{id}/_source").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(HEAD, "/{index}/{type}/{id}/_source").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/{index}/_source/{id}"), new Route(HEAD, "/{index}/_source/{id}"));
     }
 
     @Override
@@ -62,11 +56,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type"); // consume and ignore the type
-            deprecationLogger.compatibleCritical("get_source_with_types", TYPES_DEPRECATION_MESSAGE);
-        }
-
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("id"));
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
         getRequest.routing(request.param("routing"));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index 2eb0e5a1ef038..c2437dcb96fa6 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -44,12 +43,7 @@ public class RestIndexAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(POST, "/{index}/_doc/{id}"),
-            new Route(PUT, "/{index}/_doc/{id}"),
-            Route.builder(POST, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/{index}/_doc/{id}"), new Route(PUT, "/{index}/_doc/{id}"));
     }
 
     @Override
@@ -67,12 +61,7 @@ public String getName() {
 
         @Override
         public List<Route> routes() {
-            return List.of(
-                new Route(POST, "/{index}/_create/{id}"),
-                new Route(PUT, "/{index}/_create/{id}"),
-                Route.builder(POST, "/{index}/{type}/{id}/_create").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-                Route.builder(PUT, "/{index}/{type}/{id}/_create").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-            );
+            return List.of(new Route(POST, "/{index}/_create/{id}"), new Route(PUT, "/{index}/_create/{id}"));
         }
 
         @Override
@@ -101,10 +90,7 @@ public String getName() {
 
         @Override
         public List<Route> routes() {
-            return List.of(
-                new Route(POST, "/{index}/_doc"),
-                Route.builder(POST, "/{index}/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-            );
+            return List.of(new Route(POST, "/{index}/_doc"));
         }
 
         @Override
@@ -118,10 +104,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            request.param("type"); // consume and ignore the type
-        }
-
         IndexRequest indexRequest = new IndexRequest(request.param("index"));
         indexRequest.id(request.param("id"));
         indexRequest.routing(request.param("routing"));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
index 36c9653f53316..12769ef1c73e9 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -44,9 +43,7 @@ public List<Route> routes() {
             new Route(GET, "/_mget"),
             new Route(POST, "/_mget"),
             new Route(GET, "/{index}/_mget"),
-            new Route(POST, "/{index}/_mget"),
-            Route.builder(GET, "/{index}/{type}/_mget").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_mget").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_mget")
         );
     }
 
@@ -57,9 +54,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.param("type") != null) {
-            request.param("type");
-        }
         MultiGetRequest multiGetRequest = new MultiGetRequest();
         multiGetRequest.refresh(request.paramAsBoolean("refresh", multiGetRequest.refresh()));
         multiGetRequest.preference(request.param("preference"));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
index 2832b31570723..65aa1869a41e4 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
@@ -14,9 +14,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.DeprecationRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
@@ -39,9 +37,7 @@ public List<Route> routes() {
             new Route(GET, "/_mtermvectors"),
             new Route(POST, "/_mtermvectors"),
             new Route(GET, "/{index}/_mtermvectors"),
-            new Route(POST, "/{index}/_mtermvectors"),
-            Route.builder(GET, "/{index}/{type}/_mtermvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_mtermvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_mtermvectors")
         );
     }
 
@@ -52,10 +48,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type");
-            deprecationLogger.compatibleCritical(DeprecationRestHandler.DEPRECATED_ROUTE_KEY, TYPES_DEPRECATION_MESSAGE);
-        }
         MultiTermVectorsRequest multiTermVectorsRequest = new MultiTermVectorsRequest();
         TermVectorsRequest template = new TermVectorsRequest().index(request.param("index"));
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
index b2bface51ce89..8e41e1cd09674 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -44,11 +43,7 @@ public List<Route> routes() {
             new Route(GET, "/{index}/_termvectors"),
             new Route(POST, "/{index}/_termvectors"),
             new Route(GET, "/{index}/_termvectors/{id}"),
-            new Route(POST, "/{index}/_termvectors/{id}"),
-            Route.builder(GET, "/{index}/{type}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/{type}/{id}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/{id}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_termvectors/{id}")
         );
     }
 
@@ -59,11 +54,7 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type");
-        }
-        TermVectorsRequest termVectorsRequest;
-        termVectorsRequest = new TermVectorsRequest(request.param("index"), request.param("id"));
+        TermVectorsRequest termVectorsRequest = new TermVectorsRequest(request.param("index"), request.param("id"));
 
         if (request.hasContentOrSourceParam()) {
             try (XContentParser parser = request.contentOrSourceParamParser()) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
index 7e9b766fb499b..57b3a89b2303b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -33,15 +32,9 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestUpdateAction extends BaseRestHandler {
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in "
-        + "document update requests is deprecated, use the endpoint /{index}/_update/{id} instead.";
-
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(POST, "/{index}/_update/{id}"),
-            Route.builder(POST, "/{index}/{type}/{id}/_update").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/{index}/_update/{id}"));
     }
 
     @Override
@@ -51,9 +44,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type");
-        }
         UpdateRequest updateRequest = new UpdateRequest(request.param("index"), request.param("id"));
         updateRequest.routing(request.param("routing"));
         updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout()));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java
index 680860332fe74..c825a8198e6e4 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java
@@ -74,10 +74,21 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         String defaultPipeline = request.param("pipeline");
         Tuple<XContentType, BytesReference> sourceTuple = request.contentOrSourceParam();
         Map<String, Object> sourceMap = XContentHelper.convertToMap(sourceTuple.v2(), false, sourceTuple.v1()).v2();
+        Map<String, Map<String, Object>> pipelineSubstitutions = (Map<String, Map<String, Object>>) sourceMap.remove(
+            "pipeline_substitutions"
+        );
+        Map<String, Map<String, Object>> componentTemplateSubstitutions = (Map<String, Map<String, Object>>) sourceMap.remove(
+            "component_template_substitutions"
+        );
+        Map<String, Map<String, Object>> indexTemplateSubstitutions = (Map<String, Map<String, Object>>) sourceMap.remove(
+            "index_template_substitutions"
+        );
+        Object mappingAddition = sourceMap.remove("mapping_addition");
         SimulateBulkRequest bulkRequest = new SimulateBulkRequest(
-            (Map<String, Map<String, Object>>) sourceMap.remove("pipeline_substitutions"),
-            (Map<String, Map<String, Object>>) sourceMap.remove("component_template_substitutions"),
-            (Map<String, Map<String, Object>>) sourceMap.remove("index_template_substitutions")
+            pipelineSubstitutions == null ? Map.of() : pipelineSubstitutions,
+            componentTemplateSubstitutions == null ? Map.of() : componentTemplateSubstitutions,
+            indexTemplateSubstitutions == null ? Map.of() : indexTemplateSubstitutions,
+            mappingAddition == null ? Map.of() : Map.of("_doc", mappingAddition)
         );
         BytesReference transformedData = convertToBulkRequestXContentBytes(sourceMap);
         bulkRequest.add(
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java
index 5556052e79ab8..c1a55874bfc58 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -46,9 +45,7 @@ public List<Route> routes() {
             new Route(GET, "/_count"),
             new Route(POST, "/_count"),
             new Route(GET, "/{index}/_count"),
-            new Route(POST, "/{index}/_count"),
-            Route.builder(GET, "/{index}/{type}/_count").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_count").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_count")
         );
     }
 
@@ -59,10 +56,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            deprecationLogger.compatibleCritical("count_with_types", TYPES_DEPRECATION_MESSAGE);
-            request.param("type");
-        }
         SearchRequest countRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")));
         countRequest.indicesOptions(IndicesOptions.fromRequest(request, countRequest.indicesOptions()));
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).trackTotalHits(true);
@@ -94,7 +87,7 @@ public RestResponse buildResponse(SearchResponse response, XContentBuilder build
                 if (terminateAfter != DEFAULT_TERMINATE_AFTER) {
                     builder.field("terminated_early", response.isTerminatedEarly());
                 }
-                builder.field("count", response.getHits().getTotalHits().value);
+                builder.field("count", response.getHits().getTotalHits().value());
                 buildBroadcastShardsHeader(
                     builder,
                     request,
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
index 11635bee3008a..c5dc047933ea6 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.explain.ExplainResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -34,16 +33,10 @@
  */
 @ServerlessScope(value = Scope.PUBLIC)
 public class RestExplainAction extends BaseRestHandler {
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying a type in explain requests is deprecated.";
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            new Route(GET, "/{index}/_explain/{id}"),
-            new Route(POST, "/{index}/_explain/{id}"),
-            Route.builder(GET, "/{index}/{type}/{id}/_explain").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/{id}/_explain").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/{index}/_explain/{id}"), new Route(POST, "/{index}/_explain/{id}"));
     }
 
     @Override
@@ -53,11 +46,7 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type");
-        }
         ExplainRequest explainRequest = new ExplainRequest(request.param("index"), request.param("id"));
-
         explainRequest.parent(request.param("parent"));
         explainRequest.routing(request.param("routing"));
         explainRequest.preference(request.param("preference"));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index bdd7c6734172d..aeb182978e1eb 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -65,9 +64,7 @@ public List<Route> routes() {
             new Route(GET, "/_msearch"),
             new Route(POST, "/_msearch"),
             new Route(GET, "/{index}/_msearch"),
-            new Route(POST, "/{index}/_msearch"),
-            Route.builder(GET, "/{index}/{type}/_msearch").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_msearch").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_msearch")
         );
     }
 
@@ -112,10 +109,6 @@ public static MultiSearchRequest parseRequest(
         Predicate<NodeFeature> clusterSupportsFeature,
         TriFunction<String, Object, SearchRequest, Boolean> extraParamParser
     ) throws IOException {
-        if (restRequest.getRestApiVersion() == RestApiVersion.V_7 && restRequest.hasParam("type")) {
-            restRequest.param("type");
-        }
-
         MultiSearchRequest multiRequest = new MultiSearchRequest();
         IndicesOptions indicesOptions = IndicesOptions.fromRequest(restRequest, multiRequest.indicesOptions());
         multiRequest.indicesOptions(indicesOptions);
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 38157efd8a370..80a85d3b9b748 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.TransportSearchAction;
@@ -20,7 +19,6 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -89,9 +87,7 @@ public List<Route> routes() {
             new Route(GET, "/_search"),
             new Route(POST, "/_search"),
             new Route(GET, "/{index}/_search"),
-            new Route(POST, "/{index}/_search"),
-            Route.builder(GET, "/{index}/{type}/_search").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}/_search").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()
+            new Route(POST, "/{index}/_search")
         );
     }
 
@@ -103,12 +99,10 @@ public Set<String> supportedCapabilities() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
 
-        SearchRequest searchRequest;
-        if (request.hasParam("min_compatible_shard_node")) {
-            searchRequest = new SearchRequest(Version.fromString(request.param("min_compatible_shard_node")));
-        } else {
-            searchRequest = new SearchRequest();
-        }
+        SearchRequest searchRequest = new SearchRequest();
+        // access the BwC param, but just drop it
+        // this might be set by old clients
+        request.param("min_compatible_shard_node");
         /*
          * We have to pull out the call to `source().size(size)` because
          * _update_by_query and _delete_by_query uses this same parsing
@@ -171,11 +165,6 @@ public static void parseSearchRequest(
         IntConsumer setSize,
         @Nullable SearchUsageHolder searchUsageHolder
     ) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            request.param("type");
-            deprecationLogger.compatibleCritical("search_with_types", TYPES_DEPRECATION_MESSAGE);
-        }
-
         if (searchRequest.source() == null) {
             searchRequest.source(new SearchSourceBuilder());
         }
@@ -252,19 +241,8 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil
             searchSourceBuilder.from(request.paramAsInt("from", 0));
         }
         if (request.hasParam("size")) {
-            int size = request.paramAsInt("size", SearchService.DEFAULT_SIZE);
-            if (request.getRestApiVersion() == RestApiVersion.V_7 && size == -1) {
-                // we treat -1 as not-set, but deprecate it to be able to later remove this funny extra treatment
-                deprecationLogger.compatibleCritical(
-                    "search-api-size-1",
-                    "Using search size of -1 is deprecated and will be removed in future versions. "
-                        + "Instead, don't use the `size` parameter if you don't want to set it explicitly."
-                );
-            } else {
-                setSize.accept(size);
-            }
+            setSize.accept(request.paramAsInt("size", SearchService.DEFAULT_SIZE));
         }
-
         if (request.hasParam("explain")) {
             searchSourceBuilder.explain(request.paramAsBoolean("explain", null));
         }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
index 45fd6afe4fca6..7828bb956a160 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
@@ -20,6 +20,11 @@ private SearchCapabilities() {}
 
     /** Support regex and range match rules in interval queries. */
     private static final String RANGE_REGEX_INTERVAL_QUERY_CAPABILITY = "range_regexp_interval_queries";
+    /** Support synthetic source with `bit` type in `dense_vector` field when `index` is set to `false`. */
+    private static final String BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY = "bit_dense_vector_synthetic_source";
 
-    public static final Set<String> CAPABILITIES = Set.of(RANGE_REGEX_INTERVAL_QUERY_CAPABILITY);
+    public static final Set<String> CAPABILITIES = Set.of(
+        RANGE_REGEX_INTERVAL_QUERY_CAPABILITY,
+        BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY
+    );
 }
diff --git a/server/src/main/java/org/elasticsearch/script/ScoreScript.java b/server/src/main/java/org/elasticsearch/script/ScoreScript.java
index c8129717b5ccd..6c7d36ee9a436 100644
--- a/server/src/main/java/org/elasticsearch/script/ScoreScript.java
+++ b/server/src/main/java/org/elasticsearch/script/ScoreScript.java
@@ -116,6 +116,11 @@ public void setDocument(int docid) {
         this.docId = docid;
     }
 
+    /** Get the current document. */
+    public int docId() {
+        return docId;
+    }
+
     public void setScorer(Scorable scorer) {
         this.scoreSupplier = () -> {
             try {
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
index 9ac060ad063a0..f24052ef7e3a9 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
@@ -10,13 +10,11 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.TransportVersions;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
@@ -27,7 +25,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.common.collect.Iterators.single;
 import static org.elasticsearch.script.ScriptContextStats.Fields.COMPILATIONS_HISTORY;
 import static org.elasticsearch.script.ScriptStats.Fields.CACHE_EVICTIONS;
 import static org.elasticsearch.script.ScriptStats.Fields.COMPILATIONS;
@@ -193,28 +190,19 @@ public ScriptCacheStats toScriptCacheStats() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
-        return Iterators.concat(
-            ChunkedToXContentHelper.startObject(SCRIPT_STATS),
-            ChunkedToXContentHelper.field(COMPILATIONS, compilations),
-            ChunkedToXContentHelper.field(CACHE_EVICTIONS, cacheEvictions),
-            ChunkedToXContentHelper.field(COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered),
-            single((builder, params) -> {
-                if (compilationsHistory != null && compilationsHistory.areTimingsEmpty() == false) {
-                    builder.startObject(COMPILATIONS_HISTORY);
-                    compilationsHistory.toXContent(builder, params);
-                    builder.endObject();
-                }
-                if (cacheEvictionsHistory != null && cacheEvictionsHistory.areTimingsEmpty() == false) {
-                    builder.startObject(COMPILATIONS_HISTORY);
-                    cacheEvictionsHistory.toXContent(builder, params);
-                    builder.endObject();
-                }
-                return builder;
-            }),
-            ChunkedToXContentHelper.array(CONTEXTS, contextStats.iterator()),
-            ChunkedToXContentHelper.endObject()
-        );
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContent.builder(params).object(SCRIPT_STATS, ob -> {
+            ob.field(COMPILATIONS, compilations);
+            ob.field(CACHE_EVICTIONS, cacheEvictions);
+            ob.field(COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered);
+            if (compilationsHistory != null && compilationsHistory.areTimingsEmpty() == false) {
+                ob.xContentObject(COMPILATIONS_HISTORY, compilationsHistory);
+            }
+            if (cacheEvictionsHistory != null && cacheEvictionsHistory.areTimingsEmpty() == false) {
+                ob.xContentObject(COMPILATIONS_HISTORY, cacheEvictionsHistory);
+            }
+            ob.array(CONTEXTS, contextStats.iterator());
+        });
     }
 
     static final class Fields {
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
index 9dde32cc75e6a..b27019765e33b 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
@@ -12,9 +12,8 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.TermStates;
-import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.common.util.CachedSupplier;
 import org.elasticsearch.features.NodeFeature;
@@ -71,17 +70,15 @@ public int uniqueTermsCount() {
     public int matchedTermsCount() {
         final int docId = docIdSupplier.getAsInt();
         int matchedTerms = 0;
+        advancePostings(docId);
 
-        try {
-            for (PostingsEnum postingsEnum : postingsSupplier.get()) {
-                if (postingsEnum != null && postingsEnum.advance(docId) == docId && postingsEnum.freq() > 0) {
-                    matchedTerms++;
-                }
+        for (PostingsEnum postingsEnum : postingsSupplier.get()) {
+            if (postingsEnum != null && postingsEnum.docID() == docId) {
+                matchedTerms++;
             }
-            return matchedTerms;
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
         }
+
+        return matchedTerms;
     }
 
     /**
@@ -150,8 +147,9 @@ public StatsSummary termFreq() {
         final int docId = docIdSupplier.getAsInt();
 
         try {
+            advancePostings(docId);
             for (PostingsEnum postingsEnum : postingsSupplier.get()) {
-                if (postingsEnum == null || postingsEnum.advance(docId) != docId) {
+                if (postingsEnum == null || postingsEnum.docID() != docId) {
                     statsSummary.accept(0);
                 } else {
                     statsSummary.accept(postingsEnum.freq());
@@ -170,12 +168,13 @@ public StatsSummary termFreq() {
      * @return statistics on termPositions for the terms of the query in the current dac
      */
     public StatsSummary termPositions() {
-        try {
-            statsSummary.reset();
-            int docId = docIdSupplier.getAsInt();
+        statsSummary.reset();
+        int docId = docIdSupplier.getAsInt();
 
+        try {
+            advancePostings(docId);
             for (PostingsEnum postingsEnum : postingsSupplier.get()) {
-                if (postingsEnum == null || postingsEnum.advance(docId) != docId) {
+                if (postingsEnum == null || postingsEnum.docID() != docId) {
                     continue;
                 }
                 for (int i = 0; i < postingsEnum.freq(); i++) {
@@ -206,25 +205,9 @@ private TermStates[] loadTermContexts() {
     private PostingsEnum[] loadPostings() {
         try {
             PostingsEnum[] postings = new PostingsEnum[terms.length];
-            TermStates[] contexts = termContextsSupplier.get();
 
             for (int i = 0; i < terms.length; i++) {
-                TermStates termStates = contexts[i];
-                if (termStates.docFreq() == 0) {
-                    postings[i] = null;
-                    continue;
-                }
-
-                TermState state = termStates.get(leafReaderContext);
-                if (state == null) {
-                    postings[i] = null;
-                    continue;
-                }
-
-                TermsEnum termsEnum = leafReaderContext.reader().terms(terms[i].field()).iterator();
-                termsEnum.seekExact(terms[i].bytes(), state);
-
-                postings[i] = termsEnum.postings(null, PostingsEnum.ALL);
+                postings[i] = leafReaderContext.reader().postings(terms[i], PostingsEnum.POSITIONS);
             }
 
             return postings;
@@ -232,4 +215,16 @@ private PostingsEnum[] loadPostings() {
             throw new UncheckedIOException(e);
         }
     }
+
+    private void advancePostings(int targetDocId) {
+        try {
+            for (PostingsEnum posting : postingsSupplier.get()) {
+                if (posting != null && posting.docID() < targetDocId && posting.docID() != DocIdSetIterator.NO_MORE_DOCS) {
+                    posting.advance(targetDocId);
+                }
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java b/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java
index c80a2e7200ecc..d83530e82b16d 100644
--- a/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java
@@ -46,9 +46,8 @@ public void setDocument(int docID) {
     public void execute() {
         try {
             if (hasValue) {
-                long ord;
-                while ((ord = sortedSetDocValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
-                    BytesRef bytesRef = sortedSetDocValues.lookupOrd(ord);
+                for (int i = 0; i < sortedSetDocValues.docValueCount(); i++) {
+                    BytesRef bytesRef = sortedSetDocValues.lookupOrd(sortedSetDocValues.nextOrd());
                     emit(bytesRef.utf8ToString());
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/script/field/IpDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/IpDocValuesField.java
index 6297fbaa23187..d9550dd17a058 100644
--- a/server/src/main/java/org/elasticsearch/script/field/IpDocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/IpDocValuesField.java
@@ -157,7 +157,8 @@ public SortedSetIpSupplier(SortedSetDocValues in) {
         public void setNextDocId(int docId) throws IOException {
             count = 0;
             if (in.advanceExact(docId)) {
-                for (long ord = in.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = in.nextOrd()) {
+                for (int i = 0; i < in.docValueCount(); i++) {
+                    long ord = in.nextOrd();
                     ords = ArrayUtil.grow(ords, count + 1);
                     ords[count++] = ord;
                 }
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java
index fd7c5227e22ac..be1b972dcd41a 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.script.field.vectors;
 
 import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
 import org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues;
@@ -19,7 +20,8 @@
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 public class ByteKnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
-    protected ByteVectorValues input; // null if no vectors
+    protected final ByteVectorValues input; // null if no vectors
+    protected final KnnVectorValues.DocIndexIterator iterator; // null if no vectors
     protected byte[] vector;
     protected final int dims;
 
@@ -31,6 +33,7 @@ protected ByteKnnDenseVectorDocValuesField(@Nullable ByteVectorValues input, Str
         super(name, elementType);
         this.dims = dims;
         this.input = input;
+        this.iterator = input == null ? null : input.iterator();
     }
 
     @Override
@@ -38,15 +41,15 @@ public void setNextDocId(int docId) throws IOException {
         if (input == null) {
             return;
         }
-        int currentDoc = input.docID();
+        int currentDoc = iterator.docID();
         if (currentDoc == NO_MORE_DOCS || docId < currentDoc) {
             vector = null;
         } else if (docId == currentDoc) {
-            vector = input.vectorValue();
+            vector = input.vectorValue(iterator.index());
         } else {
-            currentDoc = input.advance(docId);
+            currentDoc = iterator.advance(docId);
             if (currentDoc == docId) {
-                vector = input.vectorValue();
+                vector = input.vectorValue(iterator.index());
             } else {
                 vector = null;
             }
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java
index c7678b03dd8c5..3e38092200511 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.script.field.vectors;
 
 import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenormalizedCosineFloatVectorValues;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
@@ -20,7 +21,8 @@
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 public class KnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
-    protected FloatVectorValues input; // null if no vectors
+    protected final FloatVectorValues input; // null if no vectors
+    protected final KnnVectorValues.DocIndexIterator iterator;
     protected float[] vector;
     protected final int dims;
 
@@ -28,6 +30,7 @@ public KnnDenseVectorDocValuesField(@Nullable FloatVectorValues input, String na
         super(name, ElementType.FLOAT);
         this.dims = dims;
         this.input = input;
+        this.iterator = input == null ? null : input.iterator();
     }
 
     @Override
@@ -35,15 +38,15 @@ public void setNextDocId(int docId) throws IOException {
         if (input == null) {
             return;
         }
-        int currentDoc = input.docID();
+        int currentDoc = iterator.docID();
         if (currentDoc == NO_MORE_DOCS || docId < currentDoc) {
             vector = null;
         } else if (docId == currentDoc) {
-            vector = input.vectorValue();
+            vector = input.vectorValue(iterator.index());
         } else {
-            currentDoc = input.advance(docId);
+            currentDoc = iterator.advance(docId);
             if (currentDoc == docId) {
-                vector = input.vectorValue();
+                vector = input.vectorValue(iterator.index());
             } else {
                 vector = null;
             }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 1521b17a81766..8ac35f7c40caa 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -291,7 +291,7 @@ static int determineMaximumNumberOfSlices(
         ToLongFunction<String> fieldCardinality
     ) {
         return executor instanceof ThreadPoolExecutor tpe
-            && tpe.getQueue().isEmpty()
+            && tpe.getQueue().size() <= tpe.getMaximumPoolSize()
             && isParallelCollectionSupportedForResults(resultsType, request.source(), fieldCardinality, enableQueryPhaseParallelCollection)
                 ? tpe.getMaximumPoolSize()
                 : 1;
diff --git a/server/src/main/java/org/elasticsearch/search/MultiValueMode.java b/server/src/main/java/org/elasticsearch/search/MultiValueMode.java
index 49480816bbbb1..5ac25fe0ff695 100644
--- a/server/src/main/java/org/elasticsearch/search/MultiValueMode.java
+++ b/server/src/main/java/org/elasticsearch/search/MultiValueMode.java
@@ -477,11 +477,11 @@ protected BytesRef pick(
 
         @Override
         protected int pick(SortedSetDocValues values) throws IOException {
-            long maxOrd = -1;
-            for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
-                maxOrd = ord;
+            int count = values.docValueCount();
+            for (int i = 0; i < count - 1; ++i) {
+                values.nextOrd();
             }
-            return Math.toIntExact(maxOrd);
+            return Math.toIntExact(values.nextOrd());
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java
index 6a89d66bb3411..beac39c2de304 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java
@@ -16,8 +16,11 @@
 import java.util.Set;
 
 public final class SearchFeatures implements FeatureSpecification {
+
+    public static final NodeFeature LUCENE_10_0_0_UPGRADE = new NodeFeature("lucene_10_upgrade");
+
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(KnnVectorQueryBuilder.K_PARAM_SUPPORTED);
+        return Set.of(KnnVectorQueryBuilder.K_PARAM_SUPPORTED, LUCENE_10_0_0_UPGRADE);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
index 381b4bc0e9008..1611c95d99df4 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RefCounted;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.SimpleRefCounted;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -816,9 +815,6 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t
         if (index != null) {
             builder.field(Fields._INDEX, RemoteClusterAware.buildRemoteIndexName(clusterAlias, index));
         }
-        if (builder.getRestApiVersion() == RestApiVersion.V_7 && metaFields.containsKey(MapperService.TYPE_FIELD_NAME) == false) {
-            builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME);
-        }
         if (id != null) {
             builder.field(Fields._ID, id);
         }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHits.java b/server/src/main/java/org/elasticsearch/search/SearchHits.java
index 8ff5de3c9b8ac..fe133cbac335d 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHits.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHits.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.SimpleRefCounted;
@@ -285,26 +284,22 @@ public static final class Fields {
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         assert hasReferences();
-        return Iterators.concat(Iterators.single((b, p) -> b.startObject(Fields.HITS)), Iterators.single((b, p) -> {
-            boolean totalHitAsInt = params.paramAsBoolean(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, false);
+        return ChunkedToXContent.builder(params).object(Fields.HITS, ob -> {
+            boolean totalHitAsInt = ob.params().paramAsBoolean(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, false);
             if (totalHitAsInt) {
-                long total = totalHits == null ? -1 : totalHits.value;
-                b.field(Fields.TOTAL, total);
+                ob.field(Fields.TOTAL, totalHits == null ? -1 : totalHits.value());
             } else if (totalHits != null) {
-                b.startObject(Fields.TOTAL);
-                b.field("value", totalHits.value);
-                b.field("relation", totalHits.relation == Relation.EQUAL_TO ? "eq" : "gte");
-                b.endObject();
+                ob.append((b, p) -> {
+                    b.startObject(Fields.TOTAL);
+                    b.field("value", totalHits.value());
+                    b.field("relation", totalHits.relation() == Relation.EQUAL_TO ? "eq" : "gte");
+                    return b.endObject();
+                });
             }
-            return b;
-        }), Iterators.single((b, p) -> {
-            if (Float.isNaN(maxScore)) {
-                b.nullField(Fields.MAX_SCORE);
-            } else {
-                b.field(Fields.MAX_SCORE, maxScore);
-            }
-            return b;
-        }), ChunkedToXContentHelper.array(Fields.HITS, Iterators.forArray(hits)), ChunkedToXContentHelper.endObject());
+
+            ob.field(Fields.MAX_SCORE, Float.isNaN(maxScore) ? null : maxScore);
+            ob.array(Fields.HITS, Iterators.forArray(hits));
+        });
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 6fff4318e5b35..be96b4e25d841 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -17,7 +17,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.ResolvedIndices;
@@ -33,6 +32,7 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -112,6 +112,7 @@
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
+import org.elasticsearch.search.query.SearchTimeoutException;
 import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
 import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
@@ -598,9 +599,13 @@ private <T extends RefCounted> void ensureAfterSeqNoRefreshed(
             final TimeValue timeout = request.getWaitForCheckpointsTimeout();
             final Scheduler.ScheduledCancellable timeoutTask = NO_TIMEOUT.equals(timeout) ? null : threadPool.schedule(() -> {
                 if (isDone.compareAndSet(false, true)) {
-                    listener.onFailure(
-                        new ElasticsearchTimeoutException("Wait for seq_no [{}] refreshed timed out [{}]", waitForCheckpoint, timeout)
+                    var shardTarget = new SearchShardTarget(
+                        shard.routingEntry().currentNodeId(),
+                        shard.shardId(),
+                        request.getClusterAlias()
                     );
+                    var message = LoggerMessageFormat.format("Wait for seq_no [{}] refreshed timed out [{}]", waitForCheckpoint, timeout);
+                    listener.onFailure(new SearchTimeoutException(shardTarget, message));
                 }
             }, timeout, EsExecutors.DIRECT_EXECUTOR_SERVICE);
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java
index 043fab6f4f122..b2ca4a10dc4b3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.search.aggregations.Aggregator.BucketComparator;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
@@ -591,15 +590,6 @@ public static BucketOrder parseOrderParam(XContentParser parser) throws IOExcept
             if (orderKey == null) {
                 throw new ParsingException(parser.getTokenLocation(), "Must specify at least one field for [order]");
             }
-            // _term and _time order deprecated in 6.0; replaced by _key
-            if (parser.getRestApiVersion() == RestApiVersion.V_7 && ("_term".equals(orderKey) || "_time".equals(orderKey))) {
-                deprecationLogger.compatibleCritical(
-                    "_term_and_time_key_removal",
-                    "Deprecated aggregation order key [{}] used, replaced by [_key]",
-                    orderKey
-                );
-                return orderAsc ? KEY_ASC : KEY_DESC;
-            }
             return switch (orderKey) {
                 case "_key" -> orderAsc ? KEY_ASC : KEY_DESC;
                 case "_count" -> orderAsc ? COUNT_ASC : COUNT_DESC;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java
index f66f6b4a3805d..624db3f1cfe8c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketCollector.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.MultiCollector;
 import org.apache.lucene.search.Scorable;
-import org.apache.lucene.search.ScoreCachingWrappingScorer;
 import org.apache.lucene.search.ScoreMode;
 
 import java.io.IOException;
@@ -201,6 +200,7 @@ private static class MultiLeafBucketCollector extends LeafBucketCollector {
         private final boolean cacheScores;
         private final LeafBucketCollector[] collectors;
         private int numCollectors;
+        private ScoreCachingScorable scorable;
 
         private MultiLeafBucketCollector(List<LeafBucketCollector> collectors, boolean cacheScores) {
             this.collectors = collectors.toArray(new LeafBucketCollector[collectors.size()]);
@@ -211,11 +211,11 @@ private MultiLeafBucketCollector(List<LeafBucketCollector> collectors, boolean c
         @Override
         public void setScorer(Scorable scorer) throws IOException {
             if (cacheScores) {
-                scorer = ScoreCachingWrappingScorer.wrap(scorer);
+                scorable = new ScoreCachingScorable(scorer);
             }
             for (int i = 0; i < numCollectors; ++i) {
                 final LeafCollector c = collectors[i];
-                c.setScorer(scorer);
+                c.setScorer(cacheScores ? scorable : scorer);
             }
         }
 
@@ -227,6 +227,9 @@ private void removeCollector(int i) {
 
         @Override
         public void collect(int doc, long bucket) throws IOException {
+            if (scorable != null) {
+                scorable.curDoc = doc;
+            }
             final LeafBucketCollector[] collectors = this.collectors;
             int numCollectors = this.numCollectors;
             for (int i = 0; i < numCollectors;) {
@@ -244,4 +247,25 @@ public void collect(int doc, long bucket) throws IOException {
             }
         }
     }
+
+    private static class ScoreCachingScorable extends Scorable {
+
+        private final Scorable in;
+        private int curDoc = -1; // current document
+        private int scoreDoc = -1; // document that score was computed on
+        private float score;
+
+        ScoreCachingScorable(Scorable in) {
+            this.in = in;
+        }
+
+        @Override
+        public float score() throws IOException {
+            if (curDoc != scoreDoc) {
+                score = in.score();
+                scoreDoc = curDoc;
+            }
+            return score;
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
index 2e9e04eca4afc..9ee15306ce636 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -278,7 +278,7 @@ private static boolean isMaybeMultivalued(LeafReaderContext context, SortField s
      * optimization and null if index sort is not applicable.
      */
     private Sort buildIndexSortPrefix(LeafReaderContext context) throws IOException {
-        Sort indexSort = context.reader().getMetaData().getSort();
+        Sort indexSort = context.reader().getMetaData().sort();
         if (indexSort == null) {
             return null;
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
index 0d0d2c6f922e8..dcc2ad52cbc50 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
@@ -41,8 +41,6 @@
 import java.util.List;
 import java.util.function.BiConsumer;
 
-import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
-
 /**
  * A {@link SingleDimensionValuesSource} for global ordinals.
  */
@@ -247,9 +245,8 @@ public DocIdSetIterator competitiveIterator() {
                 @Override
                 public void collect(int doc, long bucket) throws IOException {
                     if (dvs.advanceExact(doc)) {
-                        long ord;
-                        while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) {
-                            currentValue = ord;
+                        for (int i = 0; i < dvs.docValueCount(); i++) {
+                            currentValue = dvs.nextOrd();
                             next.collect(doc, bucket);
                         }
                     } else if (missingBucket) {
@@ -306,8 +303,8 @@ public void collect(int doc, long bucket) throws IOException {
                 public void collect(int doc, long bucket) throws IOException {
                     if (currentValueIsSet == false) {
                         if (dvs.advanceExact(doc)) {
-                            long ord;
-                            while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) {
+                            for (int i = 0; i < dvs.docValueCount(); i++) {
+                                long ord = dvs.nextOrd();
                                 if (term.equals(lookup.lookupOrd(ord))) {
                                     currentValueIsSet = true;
                                     currentValue = ord;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java
index f774f67b3df8f..af4d60bf424a7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java
@@ -39,7 +39,6 @@
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
-import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
 import static org.elasticsearch.search.aggregations.InternalOrder.isKeyOrder;
 
 class CountedTermsAggregator extends TermsAggregator {
@@ -77,7 +76,8 @@ private LeafBucketCollector getLeafCollector(SortedSetDocValues ords, LeafBucket
             @Override
             public void collect(int doc, long owningBucketOrd) throws IOException {
                 if (ords.advanceExact(doc)) {
-                    for (long ord = ords.nextOrd(); ord != NO_MORE_ORDS; ord = ords.nextOrd()) {
+                    for (int i = 0; i < ords.docValueCount(); i++) {
+                        long ord = ords.nextOrd();
                         collectOrdinal(bucketOrds.add(owningBucketOrd, ords.lookupOrd(ord)), doc, sub);
                     }
                 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java
index 2969b7bf82c80..7dd192b317a57 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.ConstantScoreWeight;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;
@@ -21,7 +20,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 
@@ -117,15 +115,6 @@ public int count(LeafReaderContext context) throws IOException {
                 return multiValuedSegmentWeight().count(context);
             }
 
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                ScorerSupplier scorerSupplier = scorerSupplier(context);
-                if (scorerSupplier == null) {
-                    return null;
-                }
-                return scorerSupplier.get(Long.MAX_VALUE);
-            }
-
             @Override
             public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 /*
@@ -144,19 +133,6 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
                 return multiValuedSegmentWeight().scorerSupplier(context);
             }
 
-            @Override
-            public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
-                PointValues points = context.reader().getPointValues(field);
-                if (points == null) {
-                    return null;
-                }
-                if (points.size() == points.getDocCount()) {
-                    // Each doc that has points has exactly one point.
-                    return singleValuedSegmentWeight().bulkScorer(context);
-                }
-                return multiValuedSegmentWeight().bulkScorer(context);
-            }
-
             private Weight singleValuedSegmentWeight() throws IOException {
                 if (singleValuedSegmentWeight == null) {
                     singleValuedSegmentWeight = delegateForSingleValuedSegments.createWeight(searcher, scoreMode, boost);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java
index 282c09c84414c..e8e33655d47c1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
@@ -215,7 +216,7 @@ long count(LeafReaderContext ctx, FiltersAggregator.Counter counter, Bits live)
             // No hits in this segment.
             return 0;
         }
-        scorer.score(counter, live);
+        scorer.score(counter, live, 0, DocIdSetIterator.NO_MORE_DOCS);
         return counter.readAndReset(ctx);
     }
 
@@ -228,7 +229,7 @@ void collect(LeafReaderContext ctx, LeafCollector collector, Bits live) throws I
             // No hits in this segment.
             return;
         }
-        scorer.score(collector, live);
+        scorer.score(collector, live, 0, DocIdSetIterator.NO_MORE_DOCS);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java
index 89240a94f14ce..da16d8d79fef4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java
@@ -21,8 +21,6 @@
 import java.io.IOException;
 import java.util.Locale;
 
-import static org.elasticsearch.common.geo.GeoUtils.normalizeLat;
-import static org.elasticsearch.common.geo.GeoUtils.normalizeLon;
 import static org.elasticsearch.common.geo.GeoUtils.quantizeLat;
 
 /**
@@ -113,15 +111,13 @@ public static int checkPrecisionRange(int precision) {
      * Calculates the x-coordinate in the tile grid for the specified longitude given
      * the number of tile columns for a pre-determined zoom-level.
      *
-     * @param longitude the longitude to use when determining the tile x-coordinate
+     * @param longitude the longitude to use when determining the tile x-coordinate. Longitude is in degrees
+     *                  and must be between -180 and 180 degrees.
      * @param tiles     the number of tiles per row for a pre-determined zoom-level
      */
     public static int getXTile(double longitude, int tiles) {
-        // normalizeLon treats this as 180, which is not friendly for tile mapping
-        if (longitude == -180) {
-            return 0;
-        }
-        final double xTile = (normalizeLon(longitude) + 180.0) / 360.0 * tiles;
+        assert longitude >= -180 && longitude <= 180 : "Longitude must be between -180 and 180 degrees";
+        final double xTile = (longitude + 180.0) / 360.0 * tiles;
         // Edge values may generate invalid values, and need to be clipped.
         return Math.max(0, Math.min(tiles - 1, (int) Math.floor(xTile)));
     }
@@ -130,11 +126,13 @@ public static int getXTile(double longitude, int tiles) {
      * Calculates the y-coordinate in the tile grid for the specified longitude given
      * the number of tile rows for pre-determined zoom-level.
      *
-     * @param latitude  the latitude to use when determining the tile y-coordinate
+     * @param latitude  the latitude to use when determining the tile y-coordinate. Latitude is in degrees
+     *                  and must be between -90 and 90 degrees.
      * @param tiles     the number of tiles per column for a pre-determined zoom-level
      */
     public static int getYTile(double latitude, int tiles) {
-        final double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(normalizeLat(latitude)));
+        assert latitude >= -90 && latitude <= 90 : "Latitude must be between -90 and 90 degrees";
+        final double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(latitude));
         final double yTile = (0.5 - (ESSloppyMath.log((1.0 + latSin) / (1.0 - latSin)) / PI_TIMES_4)) * tiles;
         // Edge values may generate invalid values, and need to be clipped.
         // For example, polar regions (above/below lat 85.05112878) get normalized.
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
index 4c87b5961ac1a..b5d3485e72f82 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.search.aggregations.bucket.global;
 
 import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Scorable;
@@ -45,6 +46,7 @@ public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx,
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
         grow(1);
+
         scorer.score(new LeafCollector() {
             @Override
             public void collect(int doc) throws IOException {
@@ -55,7 +57,7 @@ public void collect(int doc) throws IOException {
             public void setScorer(Scorable scorer) throws IOException {
                 sub.setScorer(scorer);
             }
-        }, aggCtx.getLeafReaderContext().reader().getLiveDocs());
+        }, aggCtx.getLeafReaderContext().reader().getLiveDocs(), 0, DocIdSetIterator.NO_MORE_DOCS);
         return LeafBucketCollector.NO_OP_COLLECTOR;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
index 6f03f9e5952cf..6886b0e22c20a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
@@ -15,22 +15,17 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.time.ZoneId;
 import java.util.Objects;
 
-import static org.elasticsearch.core.RestApiVersion.equalTo;
-
 /**
  * A class that handles all the parsing, bwc and deprecations surrounding date histogram intervals.
  *
@@ -91,29 +86,6 @@ public String getPreferredName() {
     private IntervalTypeEnum intervalType = IntervalTypeEnum.NONE;
 
     public static <T extends DateIntervalConsumer<T>> void declareIntervalFields(ObjectParser<T, String> parser) {
-        /*
-         REST version compatibility.  When in V_7 compatibility mode, continue to parse the old style interval parameter,
-         but immediately adapt it into either fixed or calendar interval.
-         */
-        parser.declareField((wrapper, interval) -> {
-            DEPRECATION_LOGGER.warn(DeprecationCategory.AGGREGATIONS, "date-interval-getter", DEPRECATION_TEXT);
-            if (interval instanceof Long) {
-                wrapper.fixedInterval(new DateHistogramInterval(interval + "ms"));
-            } else {
-                if (interval != null && DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(interval.toString())) {
-                    wrapper.calendarInterval((DateHistogramInterval) interval);
-                } else {
-                    wrapper.fixedInterval((DateHistogramInterval) interval);
-                }
-            }
-        }, p -> {
-            if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                return p.longValue();
-            } else {
-                return new DateHistogramInterval(p.text());
-            }
-        }, Histogram.INTERVAL_FIELD.forRestApiVersion(equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.LONG);
-
         parser.declareField(
             DateIntervalConsumer::calendarInterval,
             p -> new DateHistogramInterval(p.text()),
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
index 12182a5931a4f..0fbb9745aa400 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
@@ -197,7 +197,6 @@ void processBufferedChildBuckets() throws IOException {
             }
 
             for (; childDocId < currentParentDoc; childDocId = childDocs.nextDoc()) {
-                cachedScorer.doc = childDocId;
                 for (var bucket : bucketBuffer) {
                     collectBucket(sub, childDocId, bucket);
                 }
@@ -207,19 +206,12 @@ void processBufferedChildBuckets() throws IOException {
     }
 
     private static class CachedScorable extends Scorable {
-        int doc;
         float score;
 
         @Override
         public final float score() {
             return score;
         }
-
-        @Override
-        public int docID() {
-            return doc;
-        }
-
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
index 2f18d2dc1e42e..6119af3cb6a57 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
@@ -157,7 +157,8 @@ abstract static class SortedSetRangeLeafCollector extends LeafBucketCollectorBas
                 this.collector = (doc, bucket) -> {
                     if (values.advanceExact(doc)) {
                         int lo = 0;
-                        for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+                        for (int i = 0; i < values.docValueCount(); i++) {
+                            long ord = values.nextOrd();
                             lo = collect(doc, ord, bucket, lo);
                         }
                     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java
index 0cfad5ba9e0c7..37cee75c11b48 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollector.java
@@ -15,7 +15,7 @@
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopDocsCollector;
-import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.search.TopScoreDocCollectorManager;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.util.BigArrays;
@@ -99,7 +99,7 @@ public void collect(int doc, long bucket) throws IOException {
     // Designed to be overridden by subclasses that may score docs by criteria
     // other than Lucene score
     protected TopDocsCollector<? extends ScoreDoc> createTopDocsCollector(int size) throws IOException {
-        return TopScoreDocCollector.create(size, Integer.MAX_VALUE);
+        return new TopScoreDocCollectorManager(size, null, Integer.MAX_VALUE, false).newCollector();
     }
 
     // Can be overridden by subclasses that have a different priority queue implementation
@@ -214,7 +214,6 @@ class PerSegmentCollects extends Scorable {
         private final AggregationExecutionContext aggCtx;
         int maxDocId = Integer.MIN_VALUE;
         private float currentScore;
-        private int currentDocId = -1;
         private Scorable currentScorer;
 
         PerSegmentCollects(AggregationExecutionContext aggCtx) throws IOException {
@@ -249,7 +248,6 @@ public void replayRelatedMatches(List<ScoreDoc> sd) throws IOException {
                 leafCollector.setScorer(this);
 
                 currentScore = 0;
-                currentDocId = -1;
                 if (maxDocId < 0) {
                     return;
                 }
@@ -259,7 +257,6 @@ public void replayRelatedMatches(List<ScoreDoc> sd) throws IOException {
                     int rebased = scoreDoc.doc - aggCtx.getLeafReaderContext().docBase;
                     if ((rebased >= 0) && (rebased <= maxDocId)) {
                         currentScore = scoreDoc.score;
-                        currentDocId = rebased;
                         // We stored the bucket ID in Lucene's shardIndex property
                         // for convenience.
                         leafCollector.collect(rebased, scoreDoc.shardIndex);
@@ -276,11 +273,6 @@ public float score() throws IOException {
             return currentScore;
         }
 
-        @Override
-        public int docID() {
-            return currentDocId;
-        }
-
         public void collect(int docId, long parentBucket) throws IOException {
             perBucketSamples = bigArrays.grow(perBucketSamples, parentBucket + 1);
             PerParentBucketSamples sampler = perBucketSamples.get((int) parentBucket);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
index ef4101892a461..539b9440cea25 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
@@ -118,12 +118,12 @@ public long longValue() throws IOException {
                     @Override
                     public boolean advanceExact(int target) throws IOException {
                         if (globalOrds.advanceExact(target)) {
-                            value = globalOrds.nextOrd();
                             // Check there isn't a second value for this
                             // document
-                            if (globalOrds.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
+                            if (globalOrds.docValueCount() > 1) {
                                 throw new IllegalArgumentException("Sample diversifying key must be a single valued-field");
                             }
+                            value = globalOrds.nextOrd();
                             return true;
                         } else {
                             return false;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplingQuery.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplingQuery.java
index ed39f41d9daed..89fe1a53a01cc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplingQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplingQuery.java
@@ -20,6 +20,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 
 import java.io.IOException;
@@ -76,15 +77,15 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
             }
 
             @Override
-            public Scorer scorer(LeafReaderContext context) {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 final SplittableRandom random = new SplittableRandom(BitMixer.mix(hash ^ seed));
                 int maxDoc = context.reader().maxDoc();
-                return new ConstantScoreScorer(
-                    this,
+                Scorer scorer = new ConstantScoreScorer(
                     boost,
                     ScoreMode.COMPLETE_NO_SCORES,
                     new RandomSamplingIterator(maxDoc, p, random::nextInt)
                 );
+                return new DefaultScorerSupplier(scorer);
             }
         };
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
index 1b0ec8e356082..0f7c61dc9f25b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
@@ -53,7 +53,6 @@
 import java.util.function.LongPredicate;
 import java.util.function.LongUnaryOperator;
 
-import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
 import static org.elasticsearch.search.aggregations.InternalOrder.isKeyOrder;
 
 /**
@@ -167,7 +166,8 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
                     if (false == globalOrds.advanceExact(doc)) {
                         return;
                     }
-                    for (long globalOrd = globalOrds.nextOrd(); globalOrd != NO_MORE_ORDS; globalOrd = globalOrds.nextOrd()) {
+                    for (int i = 0; i < globalOrds.docValueCount(); i++) {
+                        long globalOrd = globalOrds.nextOrd();
                         collectionStrategy.collectGlobalOrd(owningBucketOrd, doc, globalOrd, sub);
                     }
                 }
@@ -179,7 +179,8 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
                 if (false == globalOrds.advanceExact(doc)) {
                     return;
                 }
-                for (long globalOrd = globalOrds.nextOrd(); globalOrd != NO_MORE_ORDS; globalOrd = globalOrds.nextOrd()) {
+                for (int i = 0; i < globalOrds.docValueCount(); i++) {
+                    long globalOrd = globalOrds.nextOrd();
                     if (false == acceptedGlobalOrdinals.test(globalOrd)) {
                         continue;
                     }
@@ -350,7 +351,8 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
                     if (false == segmentOrds.advanceExact(doc)) {
                         return;
                     }
-                    for (long segmentOrd = segmentOrds.nextOrd(); segmentOrd != NO_MORE_ORDS; segmentOrd = segmentOrds.nextOrd()) {
+                    for (int i = 0; i < segmentOrds.docValueCount(); i++) {
+                        long segmentOrd = segmentOrds.nextOrd();
                         int docCount = docCountProvider.getDocCount(doc);
                         segmentDocCounts.increment(segmentOrd + 1, docCount);
                     }
@@ -524,7 +526,8 @@ private void forEachExcludeDeletedDocs(BucketInfoConsumer consumer) throws IOExc
                         if (liveDocs == null || liveDocs.get(docId)) {  // document is not deleted
                             globalOrds = globalOrds == null ? valuesSource.globalOrdinalsValues(ctx) : globalOrds;
                             if (globalOrds.advanceExact(docId)) {
-                                for (long globalOrd = globalOrds.nextOrd(); globalOrd != NO_MORE_ORDS; globalOrd = globalOrds.nextOrd()) {
+                                for (int i = 0; i < globalOrds.docValueCount(); i++) {
+                                    long globalOrd = globalOrds.nextOrd();
                                     if (accepted.find(globalOrd) >= 0) {
                                         continue;
                                     }
@@ -634,7 +637,8 @@ void forEachExcludeDeletedDocs(long owningBucketOrd, BucketInfoConsumer consumer
                         if (liveDocs == null || liveDocs.get(docId)) {  // document is not deleted
                             globalOrds = globalOrds == null ? valuesSource.globalOrdinalsValues(ctx) : globalOrds;
                             if (globalOrds.advanceExact(docId)) {
-                                for (long globalOrd = globalOrds.nextOrd(); globalOrd != NO_MORE_ORDS; globalOrd = globalOrds.nextOrd()) {
+                                for (int i = 0; i < globalOrds.docValueCount(); i++) {
+                                    long globalOrd = globalOrds.nextOrd();
                                     if (accepted.find(globalOrd) >= 0) {
                                         continue;
                                     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
index 4d78df2704740..4bcbe08ed227c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
@@ -358,8 +358,8 @@ public IncludeExclude(
         if (exclude != null && excludeValues != null) {
             throw new IllegalArgumentException();
         }
-        this.include = include == null ? null : new RegExp(include);
-        this.exclude = exclude == null ? null : new RegExp(exclude);
+        this.include = include == null ? null : new RegExp(include, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT);
+        this.exclude = exclude == null ? null : new RegExp(exclude, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT);
         this.includeValues = includeValues;
         this.excludeValues = excludeValues;
         this.incZeroBasedPartition = 0;
@@ -529,7 +529,7 @@ private Automaton toAutomaton() {
         if (exclude != null) {
             a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
         }
-        return a;
+        return Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     public StringFilter convertToStringFilter(DocValueFormat format) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java
index 84cd869517702..05aa80f06448d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java
@@ -308,7 +308,8 @@ public void collect(int doc, long bucketOrd) throws IOException {
                     bits = new BitArray(maxOrd, bigArrays);
                     visitedOrds.set(bucketOrd, bits);
                 }
-                for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+                for (int i = 0; i < values.docValueCount(); i++) {
+                    long ord = values.nextOrd();
                     bits.set((int) ord);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java
index 32be4513f5c3e..d0685b3a09262 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java
@@ -259,8 +259,8 @@ public CompetitiveIterator competitiveIterator() {
                             @Override
                             public void collect(int doc, long bucketOrd) throws IOException {
                                 if (docValues.advanceExact(doc)) {
-                                    for (long ord = docValues.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = docValues
-                                        .nextOrd()) {
+                                    for (int i = 0; i < docValues.docValueCount(); i++) {
+                                        long ord = docValues.nextOrd();
                                         if (bits.getAndSet(ord) == false) {
                                             competitiveIterator.onVisitedOrdinal(ord);
                                         }
@@ -309,7 +309,8 @@ public void collect(int doc, long bucketOrd) throws IOException {
                 public void collect(int doc, long bucketOrd) throws IOException {
                     if (docValues.advanceExact(doc)) {
                         final BitArray bits = getNewOrExistingBitArray(bucketOrd);
-                        for (long ord = docValues.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = docValues.nextOrd()) {
+                        for (int i = 0; i < docValues.docValueCount(); i++) {
+                            long ord = docValues.nextOrd();
                             bits.set((int) ord);
                         }
                     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java
index f7f319618fa36..8ff381cbbc84d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java
@@ -135,7 +135,7 @@ public InternalAggregation get() {
                     maxScore = reduceAndFindMaxScore(aggregations, shardDocs);
                     reducedTopDocs = TopDocs.merge(from, size, shardDocs);
                 }
-                assert reducedTopDocs.totalHits.relation == Relation.EQUAL_TO;
+                assert reducedTopDocs.totalHits.relation() == Relation.EQUAL_TO;
 
                 return new InternalTopHits(
                     getName(),
@@ -262,8 +262,8 @@ public boolean equals(Object obj) {
         InternalTopHits other = (InternalTopHits) obj;
         if (from != other.from) return false;
         if (size != other.size) return false;
-        if (topDocs.topDocs.totalHits.value != other.topDocs.topDocs.totalHits.value) return false;
-        if (topDocs.topDocs.totalHits.relation != other.topDocs.topDocs.totalHits.relation) return false;
+        if (topDocs.topDocs.totalHits.value() != other.topDocs.topDocs.totalHits.value()) return false;
+        if (topDocs.topDocs.totalHits.relation() != other.topDocs.topDocs.totalHits.relation()) return false;
         if (topDocs.topDocs.scoreDocs.length != other.topDocs.topDocs.scoreDocs.length) return false;
         for (int d = 0; d < topDocs.topDocs.scoreDocs.length; d++) {
             ScoreDoc thisDoc = topDocs.topDocs.scoreDocs[d];
@@ -287,8 +287,8 @@ public int hashCode() {
         int hashCode = super.hashCode();
         hashCode = 31 * hashCode + Integer.hashCode(from);
         hashCode = 31 * hashCode + Integer.hashCode(size);
-        hashCode = 31 * hashCode + Long.hashCode(topDocs.topDocs.totalHits.value);
-        hashCode = 31 * hashCode + topDocs.topDocs.totalHits.relation.hashCode();
+        hashCode = 31 * hashCode + Long.hashCode(topDocs.topDocs.totalHits.value());
+        hashCode = 31 * hashCode + topDocs.topDocs.totalHits.relation().hashCode();
         for (int d = 0; d < topDocs.topDocs.scoreDocs.length; d++) {
             ScoreDoc doc = topDocs.topDocs.scoreDocs[d];
             hashCode = 31 * hashCode + doc.doc;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java
index d59d824bde435..90d6c298fbd23 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java
@@ -51,7 +51,7 @@ public static boolean hasValue(InternalTDigestPercentiles agg) {
     }
 
     public static boolean hasValue(InternalTopHits agg) {
-        return (agg.getHits().getTotalHits().value == 0
+        return (agg.getHits().getTotalHits().value() == 0
             && Double.isNaN(agg.getHits().getMaxScore())
             && Double.isNaN(agg.getTopDocs().maxScore)) == false;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java
index 9d8d98bc7c7cc..87d8f839dfca1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java
@@ -19,8 +19,10 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopDocsCollector;
 import org.apache.lucene.search.TopFieldCollector;
+import org.apache.lucene.search.TopFieldCollectorManager;
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.search.TopScoreDocCollectorManager;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.MaxScoreCollector;
 import org.elasticsearch.common.lucene.Lucene;
@@ -136,12 +138,14 @@ public void collect(int docId, long bucket) throws IOException {
                     // but here we create collectors ourselves and we need prevent OOM because of crazy an offset and size.
                     topN = Math.min(topN, subSearchContext.searcher().getIndexReader().maxDoc());
                     if (sort == null) {
-                        collectors = new Collectors(TopScoreDocCollector.create(topN, Integer.MAX_VALUE), null);
+                        TopScoreDocCollector topScoreDocCollector = new TopScoreDocCollectorManager(topN, null, Integer.MAX_VALUE, false)
+                            .newCollector();
+                        collectors = new Collectors(topScoreDocCollector, null);
                     } else {
                         // TODO: can we pass trackTotalHits=subSearchContext.trackTotalHits(){
                         // Note that this would require to catch CollectionTerminatedException
                         collectors = new Collectors(
-                            TopFieldCollector.create(sort.sort, topN, Integer.MAX_VALUE),
+                            new TopFieldCollectorManager(sort.sort, topN, null, Integer.MAX_VALUE, false).newCollector(),
                             subSearchContext.trackScores() ? new MaxScoreCollector() : null
                         );
                     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
index 4724bd0db05df..9b47507628dd1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
@@ -271,18 +271,17 @@ public long nextOrd() throws IOException {
                 if (hasOrds) {
                     return values.nextOrd();
                 } else {
-                    // we want to return the next missing ord but set this to
-                    // NO_MORE_ORDS so on the next call we indicate there are no
-                    // more values
-                    long ordToReturn = nextMissingOrd;
-                    nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS;
-                    return ordToReturn;
+                    return nextMissingOrd;
                 }
             }
 
             @Override
             public int docValueCount() {
-                return values.docValueCount();
+                if (hasOrds) {
+                    return values.docValueCount();
+                } else {
+                    return 1;
+                }
             }
 
             @Override
@@ -321,7 +320,11 @@ public BytesRef lookupOrd(long ord) throws IOException {
 
             @Override
             public int docValueCount() {
-                return values.docValueCount();
+                if (hasOrds) {
+                    return values.docValueCount();
+                } else {
+                    return 1;
+                }
             }
 
             @Override
@@ -339,12 +342,7 @@ public long nextOrd() throws IOException {
                         return ord + 1;
                     }
                 } else {
-                    // we want to return the next missing ord but set this to
-                    // NO_MORE_ORDS so on the next call we indicate there are no
-                    // more values
-                    long ordToReturn = nextMissingOrd;
-                    nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS;
-                    return ordToReturn;
+                    return nextMissingOrd;
                 }
             }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java
index 742d366efa7a3..472619da78622 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java
@@ -104,7 +104,7 @@ private void search(BucketCollector bucketCollector, Weight weight) throws IOExc
             Scorer scorer = weight.scorer(leaf);
             if (scorer != null) {
                 if (minimumScore != null) {
-                    scorer = new MinScoreScorer(weight, scorer, minimumScore);
+                    scorer = new MinScoreScorer(scorer, minimumScore);
                 }
                 LeafWalker leafWalker = new LeafWalker(leaf, scorer, bucketCollector, () -> tsidOrd[0]);
                 if (leafWalker.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
diff --git a/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java
index 0d11ce15b6878..eecc840e8602c 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java
@@ -63,7 +63,7 @@ public PointInTimeBuilder(BytesReference pitID) {
     }
 
     public PointInTimeBuilder(StreamInput in) throws IOException {
-        if (in.getTransportVersion().onOrAfter(TransportVersions.BINARY_PIT_ID)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             encodedId = in.readBytesReference();
         } else {
             encodedId = new BytesArray(Base64.getUrlDecoder().decode(in.readString()));
@@ -73,7 +73,7 @@ public PointInTimeBuilder(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.BINARY_PIT_ID)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBytesReference(encodedId);
         } else {
             out.writeString(Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(encodedId)));
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 9f94ec1452019..9c96319136007 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -24,8 +24,8 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -99,10 +99,11 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
     public static final ParseField TIMEOUT_FIELD = new ParseField("timeout");
     public static final ParseField TERMINATE_AFTER_FIELD = new ParseField("terminate_after");
     public static final ParseField QUERY_FIELD = new ParseField("query");
-    public static final ParseField SUB_SEARCHES_FIELD = new ParseField("sub_searches");
+    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_RELEVANCE) // remove [sub_searches] and [rank] support in 10.0
+    public static final ParseField SUB_SEARCHES_FIELD = new ParseField("sub_searches").withAllDeprecated("retriever");
+    public static final ParseField RANK_FIELD = new ParseField("rank").withAllDeprecated("retriever");
     public static final ParseField POST_FILTER_FIELD = new ParseField("post_filter");
     public static final ParseField KNN_FIELD = new ParseField("knn");
-    public static final ParseField RANK_FIELD = new ParseField("rank");
     public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score");
     public static final ParseField VERSION_FIELD = new ParseField("version");
     public static final ParseField SEQ_NO_PRIMARY_TERM_FIELD = new ParseField("seq_no_primary_term");
@@ -1341,17 +1342,7 @@ private SearchSourceBuilder parseXContent(
                 if (FROM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     from(parser.intValue());
                 } else if (SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                    int parsedSize = parser.intValue();
-                    if (parser.getRestApiVersion() == RestApiVersion.V_7 && parsedSize == -1) {
-                        // we treat -1 as not-set, but deprecate it to be able to later remove this funny extra treatment
-                        deprecationLogger.compatibleCritical(
-                            "search-api-size-1",
-                            "Using search size of -1 is deprecated and will be removed in future versions. "
-                                + "Instead, don't use the `size` parameter if you don't want to set it explicitly."
-                        );
-                    } else {
-                        size(parsedSize);
-                    }
+                    size(parser.intValue());
                 } else if (TIMEOUT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     timeout = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName());
                 } else if (TERMINATE_AFTER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -1457,99 +1448,79 @@ private SearchSourceBuilder parseXContent(
                         scriptFields.add(new ScriptField(parser));
                     }
                     searchUsage.trackSectionUsage(SCRIPT_FIELDS_FIELD.getPreferredName());
-                } else if (parser.getRestApiVersion() == RestApiVersion.V_7
-                    && INDICES_BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        deprecationLogger.compatibleCritical(
-                            "indices_boost_object_format",
-                            "Object format in indices_boost is deprecated, please use array format instead"
-                        );
+                } else if (AGGREGATIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler())
+                    || AGGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        aggregations = AggregatorFactories.parseAggregators(parser);
+                        if (aggregations.count() > 0) {
+                            searchUsage.trackSectionUsage(AGGS_FIELD.getPreferredName());
+                        }
+                    } else if (HIGHLIGHT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        highlightBuilder = HighlightBuilder.fromXContent(parser);
+                        if (highlightBuilder.fields().size() > 0) {
+                            searchUsage.trackSectionUsage(HIGHLIGHT_FIELD.getPreferredName());
+                        }
+                    } else if (SUGGEST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        suggestBuilder = SuggestBuilder.fromXContent(parser);
+                        if (suggestBuilder.getSuggestions().size() > 0) {
+                            searchUsage.trackSectionUsage(SUGGEST_FIELD.getPreferredName());
+                        }
+                    } else if (SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        sorts = new ArrayList<>(SortBuilder.fromXContent(parser));
+                    } else if (RESCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        rescoreBuilders = new ArrayList<>();
+                        rescoreBuilders.add(RescorerBuilder.parseFromXContent(parser, searchUsage::trackRescorerUsage));
+                        searchUsage.trackSectionUsage(RESCORE_FIELD.getPreferredName());
+                    } else if (EXT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        extBuilders = new ArrayList<>();
+                        String extSectionName = null;
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                             if (token == XContentParser.Token.FIELD_NAME) {
-                                currentFieldName = parser.currentName();
-                            } else if (token.isValue()) {
-                                indexBoosts.add(new IndexBoost(currentFieldName, parser.floatValue()));
+                                extSectionName = parser.currentName();
                             } else {
-                                throw new ParsingException(
-                                    parser.getTokenLocation(),
-                                    "Unknown key for a " + token + " in [" + currentFieldName + "].",
-                                    parser.getTokenLocation()
-                                );
-                            }
-                        }
-                        searchUsage.trackSectionUsage(INDICES_BOOST_FIELD.getPreferredName());
-                    } else if (AGGREGATIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler())
-                        || AGGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            aggregations = AggregatorFactories.parseAggregators(parser);
-                            if (aggregations.count() > 0) {
-                                searchUsage.trackSectionUsage(AGGS_FIELD.getPreferredName());
-                            }
-                        } else if (HIGHLIGHT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            highlightBuilder = HighlightBuilder.fromXContent(parser);
-                            if (highlightBuilder.fields().size() > 0) {
-                                searchUsage.trackSectionUsage(HIGHLIGHT_FIELD.getPreferredName());
-                            }
-                        } else if (SUGGEST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            suggestBuilder = SuggestBuilder.fromXContent(parser);
-                            if (suggestBuilder.getSuggestions().size() > 0) {
-                                searchUsage.trackSectionUsage(SUGGEST_FIELD.getPreferredName());
-                            }
-                        } else if (SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            sorts = new ArrayList<>(SortBuilder.fromXContent(parser));
-                        } else if (RESCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            rescoreBuilders = new ArrayList<>();
-                            rescoreBuilders.add(RescorerBuilder.parseFromXContent(parser, searchUsage::trackRescorerUsage));
-                            searchUsage.trackSectionUsage(RESCORE_FIELD.getPreferredName());
-                        } else if (EXT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            extBuilders = new ArrayList<>();
-                            String extSectionName = null;
-                            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                                if (token == XContentParser.Token.FIELD_NAME) {
-                                    extSectionName = parser.currentName();
-                                } else {
-                                    SearchExtBuilder searchExtBuilder = parser.namedObject(SearchExtBuilder.class, extSectionName, null);
-                                    if (searchExtBuilder.getWriteableName().equals(extSectionName) == false) {
-                                        throw new IllegalStateException(
-                                            "The parsed ["
-                                                + searchExtBuilder.getClass().getName()
-                                                + "] object has a different writeable name compared to the name of the section that "
-                                                + " it was parsed from: found ["
-                                                + searchExtBuilder.getWriteableName()
-                                                + "] expected ["
-                                                + extSectionName
-                                                + "]"
-                                        );
-                                    }
-                                    extBuilders.add(searchExtBuilder);
+                                SearchExtBuilder searchExtBuilder = parser.namedObject(SearchExtBuilder.class, extSectionName, null);
+                                if (searchExtBuilder.getWriteableName().equals(extSectionName) == false) {
+                                    throw new IllegalStateException(
+                                        "The parsed ["
+                                            + searchExtBuilder.getClass().getName()
+                                            + "] object has a different writeable name compared to the name of the section that "
+                                            + " it was parsed from: found ["
+                                            + searchExtBuilder.getWriteableName()
+                                            + "] expected ["
+                                            + extSectionName
+                                            + "]"
+                                    );
                                 }
+                                extBuilders.add(searchExtBuilder);
                             }
-                            if (extBuilders.size() > 0) {
-                                searchUsage.trackSectionUsage(EXT_FIELD.getPreferredName());
-                            }
-                        } else if (SLICE.match(currentFieldName, parser.getDeprecationHandler())) {
-                            sliceBuilder = SliceBuilder.fromXContent(parser);
-                            if (sliceBuilder.getField() != null || sliceBuilder.getId() != -1 || sliceBuilder.getMax() != -1) {
-                                searchUsage.trackSectionUsage(SLICE.getPreferredName());
-                            }
-                        } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) {
-                            collapse = CollapseBuilder.fromXContent(parser);
-                            if (collapse.getField() != null) {
-                                searchUsage.trackSectionUsage(COLLAPSE.getPreferredName());
-                            }
-                        } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
-                            pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser);
-                            searchUsage.trackSectionUsage(POINT_IN_TIME.getPreferredName());
-                        } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            runtimeMappings = parser.map();
-                            if (runtimeMappings.size() > 0) {
-                                searchUsage.trackSectionUsage(RUNTIME_MAPPINGS_FIELD.getPreferredName());
-                            }
-                        } else {
-                            throw new ParsingException(
-                                parser.getTokenLocation(),
-                                "Unknown key for a " + token + " in [" + currentFieldName + "].",
-                                parser.getTokenLocation()
-                            );
                         }
+                        if (extBuilders.size() > 0) {
+                            searchUsage.trackSectionUsage(EXT_FIELD.getPreferredName());
+                        }
+                    } else if (SLICE.match(currentFieldName, parser.getDeprecationHandler())) {
+                        sliceBuilder = SliceBuilder.fromXContent(parser);
+                        if (sliceBuilder.getField() != null || sliceBuilder.getId() != -1 || sliceBuilder.getMax() != -1) {
+                            searchUsage.trackSectionUsage(SLICE.getPreferredName());
+                        }
+                    } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) {
+                        collapse = CollapseBuilder.fromXContent(parser);
+                        if (collapse.getField() != null) {
+                            searchUsage.trackSectionUsage(COLLAPSE.getPreferredName());
+                        }
+                    } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) {
+                        pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser);
+                        searchUsage.trackSectionUsage(POINT_IN_TIME.getPreferredName());
+                    } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        runtimeMappings = parser.map();
+                        if (runtimeMappings.size() > 0) {
+                            searchUsage.trackSectionUsage(RUNTIME_MAPPINGS_FIELD.getPreferredName());
+                        }
+                    } else {
+                        throw new ParsingException(
+                            parser.getTokenLocation(),
+                            "Unknown key for a " + token + " in [" + currentFieldName + "].",
+                            parser.getTokenLocation()
+                        );
+                    }
             } else if (token == XContentParser.Token.START_ARRAY) {
                 if (STORED_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     storedFieldsContext = StoredFieldsContext.fromXContent(STORED_FIELDS_FIELD.getPreferredName(), parser);
@@ -2208,7 +2179,7 @@ public ActionRequestValidationException validate(
         boolean allowPartialSearchResults
     ) {
         if (retriever() != null) {
-            validationException = retriever().validate(this, validationException, allowPartialSearchResults);
+            validationException = retriever().validate(this, validationException, isScroll, allowPartialSearchResults);
             List<String> specified = new ArrayList<>();
             if (subSearches().isEmpty() == false) {
                 specified.add(QUERY_FIELD.getPreferredName());
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index efde60f4a0d29..534c764c07b7b 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -61,7 +61,7 @@ public ShardFetchSearchRequest(StreamInput in) throws IOException {
         shardSearchRequest = in.readOptionalWriteable(ShardSearchRequest::new);
         rescoreDocIds = new RescoreDocIds(in);
         aggregatedDfs = in.readOptionalWriteable(AggregatedDfs::new);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.RANK_DOC_IN_SHARD_FETCH_REQUEST)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.rankDocs = in.readOptionalWriteable(RankDocShardInfo::new);
         } else {
             this.rankDocs = null;
@@ -75,7 +75,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(shardSearchRequest);
         rescoreDocIds.writeTo(out);
         out.writeOptionalWriteable(aggregatedDfs);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.RANK_DOC_IN_SHARD_FETCH_REQUEST)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeOptionalWriteable(rankDocs);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java
index 719729f6787ae..f623b3040f1c5 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java
@@ -15,9 +15,7 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContent;
@@ -27,9 +25,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.core.RestApiVersion.equalTo;
-import static org.elasticsearch.core.RestApiVersion.onOrAfter;
-
 /**
  * Wrapper around a field name and the format that should be used to
  * display values of this field.
@@ -49,16 +44,7 @@ public final class FieldAndFormat implements Writeable, ToXContentObject {
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), FIELD_FIELD);
-        PARSER.declareStringOrNull(
-            ConstructingObjectParser.optionalConstructorArg(),
-            FORMAT_FIELD.forRestApiVersion(onOrAfter(RestApiVersion.V_8))
-        );
-        PARSER.declareField(
-            ConstructingObjectParser.optionalConstructorArg(),
-            ignoreUseFieldMappingStringParser(),
-            FORMAT_FIELD.forRestApiVersion(equalTo(RestApiVersion.V_7)),
-            ObjectParser.ValueType.STRING_OR_NULL
-        );
+        PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FORMAT_FIELD);
         PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), INCLUDE_UNMAPPED_FIELD);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/UnmappedFieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/UnmappedFieldFetcher.java
index 42b29fda3c472..769effdd60240 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/UnmappedFieldFetcher.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/UnmappedFieldFetcher.java
@@ -76,7 +76,7 @@ private static CharacterRunAutomaton nestedChildrenAutomaton(List<String> nested
         for (String child : nestedChildren) {
             automata.add(Operations.concatenate(Automata.makeString(child + "."), Automata.makeAnyString()));
         }
-        return new CharacterRunAutomaton(Operations.union(automata));
+        return new CharacterRunAutomaton(Operations.determinize(Operations.union(automata), AUTOMATON_MAX_DETERMINIZED_STATES));
     }
 
     // Builds an automaton that will match any field that conforms to one of the input patterns
@@ -84,7 +84,11 @@ private static CharacterRunAutomaton buildUnmappedFieldPatternAutomaton(List<Str
         if (patterns.isEmpty()) {
             return null;
         }
-        return new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(patterns.toArray(String[]::new)), AUTOMATON_MAX_DETERMINIZED_STATES);
+        Automaton a = Operations.determinize(
+            Regex.simpleMatchToAutomaton(patterns.toArray(String[]::new)),
+            AUTOMATON_MAX_DETERMINIZED_STATES
+        );
+        return new CharacterRunAutomaton(a);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java
index 1030bd87dbd14..abfcf95472765 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java
@@ -40,12 +40,12 @@ public static WeightedFragInfo fixWeightedFragInfo(WeightedFragInfo fragInfo) {
              * than in this hack... aka. "we are are working on in!" */
             final List<SubInfo> subInfos = fragInfo.getSubInfos();
             CollectionUtil.introSort(subInfos, (o1, o2) -> {
-                int startOffset = o1.getTermsOffsets().get(0).getStartOffset();
-                int startOffset2 = o2.getTermsOffsets().get(0).getStartOffset();
+                int startOffset = o1.termsOffsets().get(0).getStartOffset();
+                int startOffset2 = o2.termsOffsets().get(0).getStartOffset();
                 return Integer.compare(startOffset, startOffset2);
             });
             return new WeightedFragInfo(
-                Math.min(fragInfo.getSubInfos().get(0).getTermsOffsets().get(0).getStartOffset(), fragInfo.getStartOffset()),
+                Math.min(fragInfo.getSubInfos().get(0).termsOffsets().get(0).getStartOffset(), fragInfo.getStartOffset()),
                 fragInfo.getEndOffset(),
                 subInfos,
                 fragInfo.getTotalBoost()
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index 18de4b81cbf8c..78d90377cdc3f 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -56,6 +56,7 @@
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executor;
+import java.util.stream.Collectors;
 
 /**
  * Context-aware extension of {@link IndexSearcher}.
@@ -76,6 +77,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
     private QueryProfiler profiler;
     private final MutableQueryTimeout cancellable;
 
+    private final boolean hasExecutor;
     private final int maximumNumberOfSlices;
     // don't create slices with less than this number of docs
     private final int minimumDocsPerSlice;
@@ -133,6 +135,7 @@ public ContextIndexSearcher(
         int minimumDocsPerSlice
     ) throws IOException {
         super(wrapWithExitableDirectoryReader ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader, executor);
+        this.hasExecutor = executor != null;
         setSimilarity(similarity);
         setQueryCache(queryCache);
         setQueryCachingPolicy(queryCachingPolicy);
@@ -141,6 +144,15 @@ public ContextIndexSearcher(
         this.maximumNumberOfSlices = maximumNumberOfSlices;
     }
 
+    /**
+     * Whether an executor was provided at construction time or not. This indicates whether operations that support concurrency
+     * may be executed concurrently. It is not straightforward to deduct this from {@link #getTaskExecutor()} because {@link IndexSearcher}
+     * creates a {@link org.apache.lucene.search.TaskExecutor} anyways.
+     */
+    public boolean hasExecutor() {
+        return hasExecutor;
+    }
+
     @Override
     protected LeafSlice[] slices(List<LeafReaderContext> leaves) {
         // we offload to the executor unconditionally, including requests that don't support concurrency
@@ -149,11 +161,6 @@ protected LeafSlice[] slices(List<LeafReaderContext> leaves) {
         return leafSlices;
     }
 
-    // package private for testing
-    int getMinimumDocsPerSlice() {
-        return minimumDocsPerSlice;
-    }
-
     public void setProfiler(QueryProfiler profiler) {
         this.profiler = profiler;
     }
@@ -243,7 +250,14 @@ public static LeafSlice[] computeSlices(List<LeafReaderContext> leaves, int maxS
             throw new IllegalArgumentException("maxSliceNum must be >= 1 (got " + maxSliceNum + ")");
         }
         if (maxSliceNum == 1) {
-            return new LeafSlice[] { new LeafSlice(new ArrayList<>(leaves)) };
+            return new LeafSlice[] {
+                new LeafSlice(
+                    new ArrayList<>(
+                        leaves.stream()
+                            .map(LeafReaderContextPartition::createForEntireSegment)
+                            .collect(Collectors.toCollection(ArrayList::new))
+                    )
+                ) };
         }
         // total number of documents to be searched
         final int numDocs = leaves.stream().mapToInt(l -> l.reader().maxDoc()).sum();
@@ -291,7 +305,11 @@ private static LeafSlice[] computeSlices(List<LeafReaderContext> leaves, int min
         for (List<LeafReaderContext> currentLeaf : queue) {
             // LeafSlice ctor reorders leaves so that leaves within a slice preserve the order they had within the IndexReader.
             // This is important given how Elasticsearch sorts leaves by descending @timestamp to get better query performance.
-            slices[upto++] = new LeafSlice(currentLeaf);
+            slices[upto++] = new LeafSlice(
+                currentLeaf.stream()
+                    .map(LeafReaderContextPartition::createForEntireSegment)
+                    .collect(Collectors.toCollection(ArrayList::new))
+            );
         }
 
         return slices;
@@ -344,10 +362,10 @@ private <C extends Collector, T> T search(Weight weight, CollectorManager<C, T>
             }
             final List<Callable<C>> listTasks = new ArrayList<>(leafSlices.length);
             for (int i = 0; i < leafSlices.length; ++i) {
-                final LeafReaderContext[] leaves = leafSlices[i].leaves;
+                final LeafReaderContextPartition[] leaves = leafSlices[i].partitions;
                 final C collector = collectors.get(i);
                 listTasks.add(() -> {
-                    search(Arrays.asList(leaves), weight, collector);
+                    search(leaves, weight, collector);
                     return collector;
                 });
             }
@@ -364,7 +382,7 @@ private <C extends Collector, T> T search(Weight weight, CollectorManager<C, T>
      * 2) handles the ES TimeExceededException
      */
     @Override
-    public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
+    public void search(LeafReaderContextPartition[] leaves, Weight weight, Collector collector) throws IOException {
         boolean success = false;
         try {
             super.search(leaves, weight, collector);
@@ -407,12 +425,12 @@ public void throwTimeExceededException() {
         }
     }
 
-    private static class TimeExceededException extends RuntimeException {
+    public static class TimeExceededException extends RuntimeException {
         // This exception should never be re-thrown, but we fill in the stacktrace to be able to trace where it does not get properly caught
     }
 
     @Override
-    protected void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
+    protected void searchLeaf(LeafReaderContext ctx, int minDocId, int maxDocId, Weight weight, Collector collector) throws IOException {
         cancellable.checkCancelled();
         final LeafCollector leafCollector;
         try {
@@ -432,7 +450,7 @@ protected void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collec
                     bulkScorer = new CancellableBulkScorer(bulkScorer, cancellable::checkCancelled);
                 }
                 try {
-                    bulkScorer.score(leafCollector, liveDocs);
+                    bulkScorer.score(leafCollector, liveDocs, minDocId, maxDocId);
                 } catch (CollectionTerminatedException e) {
                     // collection was terminated prematurely
                     // continue with the following leaf
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
index 3bdd7ff3630cf..64b54d3623f04 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
@@ -14,9 +14,9 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
-import org.apache.lucene.index.FilterVectorValues;
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.QueryTimeout;
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  * Wraps an {@link IndexReader} with a {@link QueryCancellation}
@@ -459,7 +460,6 @@ public void grow(int count) {
     }
 
     private static class ExitableByteVectorValues extends ByteVectorValues {
-        private int calls;
         private final QueryCancellation queryCancellation;
         private final ByteVectorValues in;
 
@@ -479,8 +479,13 @@ public int size() {
         }
 
         @Override
-        public byte[] vectorValue() throws IOException {
-            return in.vectorValue();
+        public byte[] vectorValue(int ord) throws IOException {
+            return in.vectorValue(ord);
+        }
+
+        @Override
+        public int ordToDoc(int ord) {
+            return in.ordToDoc(ord);
         }
 
         @Override
@@ -505,33 +510,17 @@ public DocIdSetIterator iterator() {
         }
 
         @Override
-        public int docID() {
-            return in.docID();
-        }
-
-        @Override
-        public int nextDoc() throws IOException {
-            final int nextDoc = in.nextDoc();
-            checkAndThrowWithSampling();
-            return nextDoc;
+        public DocIndexIterator iterator() {
+            return createExitableIterator(in.iterator(), queryCancellation);
         }
 
         @Override
-        public int advance(int target) throws IOException {
-            final int advance = in.advance(target);
-            checkAndThrowWithSampling();
-            return advance;
-        }
-
-        private void checkAndThrowWithSampling() {
-            if ((calls++ & ExitableIntersectVisitor.MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) {
-                this.queryCancellation.checkCancelled();
-            }
+        public ByteVectorValues copy() throws IOException {
+            return in.copy();
         }
     }
 
-    private static class ExitableFloatVectorValues extends FilterVectorValues {
-        private int calls;
+    private static class ExitableFloatVectorValues extends FilterFloatVectorValues {
         private final QueryCancellation queryCancellation;
 
         ExitableFloatVectorValues(FloatVectorValues vectorValues, QueryCancellation queryCancellation) {
@@ -541,17 +530,13 @@ private static class ExitableFloatVectorValues extends FilterVectorValues {
         }
 
         @Override
-        public int advance(int target) throws IOException {
-            final int advance = super.advance(target);
-            checkAndThrowWithSampling();
-            return advance;
+        public float[] vectorValue(int ord) throws IOException {
+            return in.vectorValue(ord);
         }
 
         @Override
-        public int nextDoc() throws IOException {
-            final int nextDoc = super.nextDoc();
-            checkAndThrowWithSampling();
-            return nextDoc;
+        public int ordToDoc(int ord) {
+            return in.ordToDoc(ord);
         }
 
         @Override
@@ -575,13 +560,61 @@ public DocIdSetIterator iterator() {
             };
         }
 
-        private void checkAndThrowWithSampling() {
-            if ((calls++ & ExitableIntersectVisitor.MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) {
-                this.queryCancellation.checkCancelled();
-            }
+        @Override
+        public DocIndexIterator iterator() {
+            return createExitableIterator(in.iterator(), queryCancellation);
+        }
+
+        @Override
+        public FloatVectorValues copy() throws IOException {
+            return in.copy();
         }
     }
 
+    private static KnnVectorValues.DocIndexIterator createExitableIterator(
+        KnnVectorValues.DocIndexIterator delegate,
+        QueryCancellation queryCancellation
+    ) {
+        return new KnnVectorValues.DocIndexIterator() {
+            private int calls;
+
+            @Override
+            public int index() {
+                return delegate.index();
+            }
+
+            @Override
+            public int docID() {
+                return delegate.docID();
+            }
+
+            @Override
+            public long cost() {
+                return delegate.cost();
+            }
+
+            @Override
+            public int nextDoc() throws IOException {
+                int nextDoc = delegate.nextDoc();
+                checkAndThrowWithSampling();
+                return nextDoc;
+            }
+
+            @Override
+            public int advance(int target) throws IOException {
+                final int advance = delegate.advance(target);
+                checkAndThrowWithSampling();
+                return advance;
+            }
+
+            private void checkAndThrowWithSampling() {
+                if ((calls++ & ExitableIntersectVisitor.MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) {
+                    queryCancellation.checkCancelled();
+                }
+            }
+        };
+    }
+
     private static class ExitableDocSetIterator extends DocIdSetIterator {
         private int calls;
         private final DocIdSetIterator in;
@@ -622,4 +655,43 @@ private void checkAndThrowWithSampling() {
             }
         }
     }
+
+    /** Delegates all methods to a wrapped {@link FloatVectorValues}. */
+    private abstract static class FilterFloatVectorValues extends FloatVectorValues {
+
+        /** Wrapped values */
+        protected final FloatVectorValues in;
+
+        /** Sole constructor */
+        protected FilterFloatVectorValues(FloatVectorValues in) {
+            Objects.requireNonNull(in);
+            this.in = in;
+        }
+
+        @Override
+        public DocIndexIterator iterator() {
+            return in.iterator();
+        }
+
+        @Override
+        public float[] vectorValue(int ord) throws IOException {
+            return in.vectorValue(ord);
+        }
+
+        @Override
+        public FloatVectorValues copy() throws IOException {
+            return in.copy();
+        }
+
+        @Override
+        public int dimension() {
+            return in.dimension();
+        }
+
+        @Override
+        public int size() {
+            return in.size();
+        }
+
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java
index 9b594e2935504..f03be3f09b7d2 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java
@@ -103,15 +103,6 @@ static final class FieldUsageTrackingLeafReader extends SequentialStoredFieldsLe
             this.notifier = notifier;
         }
 
-        @Override
-        public Fields getTermVectors(int docID) throws IOException {
-            Fields f = super.getTermVectors(docID);
-            if (f != null) {
-                f = new FieldUsageTrackingTermVectorFields(f);
-            }
-            return f;
-        }
-
         @Override
         public TermVectors termVectors() throws IOException {
             TermVectors termVectors = super.termVectors();
@@ -136,11 +127,6 @@ public PointValues getPointValues(String field) throws IOException {
             return pointValues;
         }
 
-        @Override
-        public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {
-            storedFields().document(docID, visitor);
-        }
-
         @Override
         public StoredFields storedFields() throws IOException {
             StoredFields storedFields = super.storedFields();
diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
index cd8f381e85f83..f559325063bef 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.search.Weight;
 import org.elasticsearch.search.profile.Timer;
 
 import java.io.IOException;
@@ -25,15 +24,12 @@
 final class ProfileScorer extends Scorer {
 
     private final Scorer scorer;
-    private final ProfileWeight profileWeight;
 
     private final Timer scoreTimer, nextDocTimer, advanceTimer, matchTimer, shallowAdvanceTimer, computeMaxScoreTimer,
         setMinCompetitiveScoreTimer;
 
-    ProfileScorer(ProfileWeight w, Scorer scorer, QueryProfileBreakdown profile) {
-        super(w);
+    ProfileScorer(Scorer scorer, QueryProfileBreakdown profile) {
         this.scorer = scorer;
-        this.profileWeight = w;
         scoreTimer = profile.getNewTimer(QueryTimingType.SCORE);
         nextDocTimer = profile.getNewTimer(QueryTimingType.NEXT_DOC);
         advanceTimer = profile.getNewTimer(QueryTimingType.ADVANCE);
@@ -58,11 +54,6 @@ public float score() throws IOException {
         }
     }
 
-    @Override
-    public Weight getWeight() {
-        return profileWeight;
-    }
-
     @Override
     public Collection<ChildScorable> getChildren() throws IOException {
         return scorer.getChildren();
diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
index 27bf8ea8aae47..5d35699adec95 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
@@ -37,15 +37,6 @@ public ProfileWeight(Query query, Weight subQueryWeight, QueryProfileBreakdown p
         this.profile = profile;
     }
 
-    @Override
-    public Scorer scorer(LeafReaderContext context) throws IOException {
-        ScorerSupplier supplier = scorerSupplier(context);
-        if (supplier == null) {
-            return null;
-        }
-        return supplier.get(Long.MAX_VALUE);
-    }
-
     @Override
     public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
         final Timer timer = profile.getNewTimer(QueryTimingType.BUILD_SCORER);
@@ -67,12 +58,24 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
             public Scorer get(long loadCost) throws IOException {
                 timer.start();
                 try {
-                    return new ProfileScorer(weight, subQueryScorerSupplier.get(loadCost), profile);
+                    return new ProfileScorer(subQueryScorerSupplier.get(loadCost), profile);
                 } finally {
                     timer.stop();
                 }
             }
 
+            @Override
+            public BulkScorer bulkScorer() throws IOException {
+                // We use the default bulk scorer instead of the specialized one. The reason
+                // is that Lucene's BulkScorers do everything at once: finding matches,
+                // scoring them and calling the collector, so they make it impossible to
+                // see where time is spent, which is the purpose of query profiling.
+                // The default bulk scorer will pull a scorer and iterate over matches,
+                // this might be a significantly different execution path for some queries
+                // like disjunctions, but in general this is what is done anyway
+                return super.bulkScorer();
+            }
+
             @Override
             public long cost() {
                 timer.start();
@@ -90,18 +93,6 @@ public void setTopLevelScoringClause() throws IOException {
         };
     }
 
-    @Override
-    public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
-        // We use the default bulk scorer instead of the specialized one. The reason
-        // is that Lucene's BulkScorers do everything at once: finding matches,
-        // scoring them and calling the collector, so they make it impossible to
-        // see where time is spent, which is the purpose of query profiling.
-        // The default bulk scorer will pull a scorer and iterate over matches,
-        // this might be a significantly different execution path for some queries
-        // like disjunctions, but in general this is what is done anyway
-        return super.bulkScorer(context);
-    }
-
     @Override
     public Explanation explain(LeafReaderContext context, int doc) throws IOException {
         return subQueryWeight.explain(context, doc);
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index d17cd4f69dec7..af65c30b49dcf 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -248,7 +248,7 @@ private static boolean canEarlyTerminate(IndexReader reader, SortAndFormats sort
         }
         final Sort sort = sortAndFormats.sort;
         for (LeafReaderContext ctx : reader.leaves()) {
-            Sort indexSort = ctx.reader().getMetaData().getSort();
+            Sort indexSort = ctx.reader().getMetaData().sort();
             if (indexSort == null || Lucene.canEarlyTerminate(sort, indexSort) == false) {
                 return false;
             }
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollector.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollector.java
index d1cbdd6adb761..00cf90fe12301 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollector.java
@@ -202,7 +202,11 @@ public DocIdSetIterator competitiveIterator() throws IOException {
                 }
             };
         }
-        return new CompositeLeafCollector(postFilterBits, topDocsLeafCollector, aggsLeafCollector);
+        LeafCollector leafCollector = new CompositeLeafCollector(postFilterBits, topDocsLeafCollector, aggsLeafCollector);
+        if (cacheScores && topDocsLeafCollector != null && aggsLeafCollector != null) {
+            leafCollector = ScoreCachingWrappingScorer.wrap(leafCollector);
+        }
+        return leafCollector;
     }
 
     private static FilterScorable wrapToIgnoreMinCompetitiveScore(Scorable scorer) {
@@ -263,9 +267,6 @@ private class CompositeLeafCollector implements LeafCollector {
 
         @Override
         public void setScorer(Scorable scorer) throws IOException {
-            if (cacheScores && topDocsLeafCollector != null && aggsLeafCollector != null) {
-                scorer = ScoreCachingWrappingScorer.wrap(scorer);
-            }
             // Ignore calls to setMinCompetitiveScore so that if the top docs collector
             // wants to skip low-scoring hits, the aggs collector still sees all hits.
             // this is important also for terminate_after in case used when total hits tracking is early terminated.
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java
index eb8d03bd1406f..13cf9f339d7d4 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java
@@ -43,7 +43,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.RANK_FEATURE_PHASE_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java b/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
index 5cd947a1cc73b..cb9169dbeb5e5 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
@@ -26,6 +26,7 @@
 
 public final class QueryRescorer implements Rescorer {
 
+    private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10;
     public static final Rescorer INSTANCE = new QueryRescorer();
 
     @Override
@@ -39,9 +40,14 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
         final QueryRescoreContext rescore = (QueryRescoreContext) rescoreContext;
 
         org.apache.lucene.search.Rescorer rescorer = new org.apache.lucene.search.QueryRescorer(rescore.parsedQuery().query()) {
+            int count = 0;
 
             @Override
             protected float combine(float firstPassScore, boolean secondPassMatches, float secondPassScore) {
+                if (count % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) {
+                    rescore.checkCancellation();
+                }
+                count++;
                 if (secondPassMatches) {
                     return rescore.scoreMode.combine(
                         firstPassScore * rescore.queryWeight(),
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java b/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java
index 297b197a6d0c1..0ae6c326ddcdc 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java
@@ -24,6 +24,7 @@ public class RescoreContext {
     private final int windowSize;
     private final Rescorer rescorer;
     private Set<Integer> rescoredDocs; // doc Ids for which rescoring was applied
+    private Runnable isCancelled;
 
     /**
      * Build the context.
@@ -34,6 +35,16 @@ public RescoreContext(int windowSize, Rescorer rescorer) {
         this.rescorer = rescorer;
     }
 
+    public void setCancellationChecker(Runnable isCancelled) {
+        this.isCancelled = isCancelled;
+    }
+
+    public void checkCancellation() {
+        if (isCancelled != null) {
+            isCancelled.run();
+        }
+    }
+
     /**
      * The rescorer to actually apply.
      */
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
index be961b8ef942b..1227db5d8e1db 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
@@ -14,12 +14,18 @@
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.lucene.grouping.TopFieldGroups;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.query.QueryPhase;
+import org.elasticsearch.search.query.SearchTimeoutException;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -44,11 +50,14 @@ public static void execute(SearchContext context) {
             topGroups = topFieldGroups;
         }
         try {
+            Runnable cancellationCheck = getCancellationChecks(context);
             for (RescoreContext ctx : context.rescore()) {
+                ctx.setCancellationChecker(cancellationCheck);
                 topDocs = ctx.rescorer().rescore(topDocs, context.searcher(), ctx);
                 // It is the responsibility of the rescorer to sort the resulted top docs,
                 // here we only assert that this condition is met.
                 assert context.sort() == null && topDocsSortedByScore(topDocs) : "topdocs should be sorted after rescore";
+                ctx.setCancellationChecker(null);
             }
             if (topGroups != null) {
                 assert context.collapse() != null;
@@ -63,6 +72,11 @@ public static void execute(SearchContext context) {
                 .topDocs(new TopDocsAndMaxScore(topDocs, topDocs.scoreDocs[0].score), context.queryResult().sortValueFormats());
         } catch (IOException e) {
             throw new ElasticsearchException("Rescore Phase Failed", e);
+        } catch (ContextIndexSearcher.TimeExceededException e) {
+            if (context.request().allowPartialSearchResults() == false) {
+                throw new SearchTimeoutException(context.shardTarget(), "Time exceeded");
+            }
+            context.queryResult().searchTimedOut(true);
         }
     }
 
@@ -106,4 +120,27 @@ private static boolean topDocsSortedByScore(TopDocs topDocs) {
         }
         return true;
     }
+
+    static Runnable getCancellationChecks(SearchContext context) {
+        List<Runnable> cancellationChecks = new ArrayList<>();
+        if (context.lowLevelCancellation()) {
+            cancellationChecks.add(() -> {
+                final SearchShardTask task = context.getTask();
+                if (task != null) {
+                    task.ensureNotCancelled();
+                }
+            });
+        }
+
+        final Runnable timeoutRunnable = QueryPhase.getTimeoutCheck(context);
+        if (timeoutRunnable != null) {
+            cancellationChecks.add(timeoutRunnable);
+        }
+
+        return () -> {
+            for (var check : cancellationChecks) {
+                check.run();
+            }
+        };
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java
index 22bef026523e9..b15798db95b6f 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java
@@ -11,6 +11,8 @@
 
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.search.MultiSearchRequest;
@@ -18,9 +20,9 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.TransportMultiSearchAction;
-import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
@@ -122,10 +124,17 @@ public final RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOExceptio
                 public void onResponse(MultiSearchResponse items) {
                     List<ScoreDoc[]> topDocs = new ArrayList<>();
                     List<Exception> failures = new ArrayList<>();
+                    // capture the max status code returned by any of the responses
+                    int statusCode = RestStatus.OK.getStatus();
+                    List<String> retrieversWithFailures = new ArrayList<>();
                     for (int i = 0; i < items.getResponses().length; i++) {
                         var item = items.getResponses()[i];
                         if (item.isFailure()) {
                             failures.add(item.getFailure());
+                            retrieversWithFailures.add(innerRetrievers.get(i).retriever().getName());
+                            if (ExceptionsHelper.status(item.getFailure()).getStatus() > statusCode) {
+                                statusCode = ExceptionsHelper.status(item.getFailure()).getStatus();
+                            }
                         } else {
                             assert item.getResponse() != null;
                             var rankDocs = getRankDocs(item.getResponse());
@@ -134,7 +143,14 @@ public void onResponse(MultiSearchResponse items) {
                         }
                     }
                     if (false == failures.isEmpty()) {
-                        IllegalStateException ex = new IllegalStateException("Search failed - some nested retrievers returned errors.");
+                        assert statusCode != RestStatus.OK.getStatus();
+                        final String errMessage = "["
+                            + getName()
+                            + "] search failed - retrievers '"
+                            + retrieversWithFailures
+                            + "' returned errors. "
+                            + "All failures are attached as suppressed exceptions.";
+                        Exception ex = new ElasticsearchStatusException(errMessage, RestStatus.fromCode(statusCode));
                         failures.forEach(ex::addSuppressed);
                         listener.onFailure(ex);
                     } else {
@@ -163,6 +179,11 @@ public final QueryBuilder topDocsQuery() {
         throw new IllegalStateException("Should not be called, missing a rewrite?");
     }
 
+    @Override
+    public final QueryBuilder explainQuery() {
+        throw new IllegalStateException("Should not be called, missing a rewrite?");
+    }
+
     @Override
     public final void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
         throw new IllegalStateException("Should not be called, missing a rewrite?");
@@ -172,9 +193,10 @@ public final void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceB
     public ActionRequestValidationException validate(
         SearchSourceBuilder source,
         ActionRequestValidationException validationException,
+        boolean isScroll,
         boolean allowPartialSearchResults
     ) {
-        validationException = super.validate(source, validationException, allowPartialSearchResults);
+        validationException = super.validate(source, validationException, isScroll, allowPartialSearchResults);
         if (source.size() > rankWindowSize) {
             validationException = addValidationError(
                 "["
@@ -190,10 +212,16 @@ public ActionRequestValidationException validate(
         }
         if (allowPartialSearchResults) {
             validationException = addValidationError(
-                "cannot specify a compound retriever and [allow_partial_search_results]",
+                "cannot specify [" + getName() + "] and [allow_partial_search_results]",
                 validationException
             );
         }
+        if (isScroll) {
+            validationException = addValidationError("cannot specify [" + getName() + "] and [scroll]", validationException);
+        }
+        for (RetrieverSource innerRetriever : innerRetrievers) {
+            validationException = innerRetriever.retriever().validate(source, validationException, isScroll, allowPartialSearchResults);
+        }
         return validationException;
     }
 
@@ -213,22 +241,12 @@ protected SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit,
             .trackTotalHits(false)
             .storedFields(new StoredFieldsContext(false))
             .size(rankWindowSize);
+        // apply the pre-filters downstream once
         if (preFilterQueryBuilders.isEmpty() == false) {
             retrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders);
         }
         retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, true);
 
-        // apply the pre-filters
-        if (preFilterQueryBuilders.size() > 0) {
-            QueryBuilder query = sourceBuilder.query();
-            BoolQueryBuilder newQuery = new BoolQueryBuilder();
-            if (query != null) {
-                newQuery.must(query);
-            }
-            preFilterQueryBuilders.forEach(newQuery::filter);
-            sourceBuilder.query(newQuery);
-        }
-
         // Record the shard id in the sort result
         List<SortBuilder<?>> sortBuilders = sourceBuilder.sorts() != null ? new ArrayList<>(sourceBuilder.sorts()) : new ArrayList<>();
         if (sortBuilders.isEmpty()) {
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
index 882d44adb79c3..5e36ad0fd4fd6 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
@@ -239,6 +239,7 @@ public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
     public ActionRequestValidationException validate(
         SearchSourceBuilder source,
         ActionRequestValidationException validationException,
+        boolean isScroll,
         boolean allowPartialSearchResults
     ) {
         return validationException;
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java
index fb5015a82dbdb..2cb960e7e73cb 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
@@ -106,13 +105,12 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 }
 
                 @Override
-                public Scorer scorer(LeafReaderContext context) {
-                    // Segment starts indicate how many docs are in the segment,
-                    // upper equalling lower indicates no documents for this segment
-                    if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) {
-                        return null;
-                    }
-                    return new Scorer(this) {
+                public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                    /**
+                     * We return a scorer even if there are no ranked documents within the segment.
+                     * This ensures the correct propagation of the maximum score.
+                     */
+                    Scorer scorer = new Scorer() {
                         final int lower = segmentStarts[context.ord];
                         final int upper = segmentStarts[context.ord + 1];
                         int upTo = -1;
@@ -181,6 +179,7 @@ private int currentDocId() {
                         }
 
                     };
+                    return new DefaultScorerSupplier(scorer);
                 }
 
                 @Override
@@ -326,11 +325,6 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 return topWeight.explain(context, doc);
             }
 
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                return combinedWeight.scorer(context);
-            }
-
             @Override
             public boolean isCacheable(LeafReaderContext ctx) {
                 return combinedWeight.isCacheable(ctx);
@@ -341,11 +335,6 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException {
                 return combinedWeight.matches(context, doc);
             }
 
-            @Override
-            public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
-                return combinedWeight.bulkScorer(context);
-            }
-
             @Override
             public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 return combinedWeight.scorerSupplier(context);
diff --git a/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java
index 5077d68c12baa..c65c2bb6650c1 100644
--- a/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/runtime/AbstractScriptFieldQuery.java
@@ -19,6 +19,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.script.AbstractFieldScript;
@@ -69,10 +70,11 @@ public boolean isCacheable(LeafReaderContext ctx) {
             }
 
             @Override
-            public Scorer scorer(LeafReaderContext ctx) {
+            public ScorerSupplier scorerSupplier(LeafReaderContext ctx) throws IOException {
                 S scriptContext = scriptContextFunction.apply(ctx);
                 DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc());
-                return new ConstantScoreScorer(this, score(), scoreMode, createTwoPhaseIterator(scriptContext, approximation));
+                Scorer scorer = new ConstantScoreScorer(score(), scoreMode, createTwoPhaseIterator(scriptContext, approximation));
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java
index 751ecb18cc68f..430d22ebc9084 100644
--- a/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQuery.java
@@ -18,6 +18,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.SloppyMath;
@@ -79,8 +80,9 @@ public boolean isCacheable(LeafReaderContext ctx) {
             }
 
             @Override
-            public Scorer scorer(LeafReaderContext context) {
-                return new DistanceScorer(this, scriptContextFunction().apply(context), context.reader().maxDoc(), boost);
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                Scorer scorer = new DistanceScorer(scriptContextFunction().apply(context), context.reader().maxDoc(), boost);
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override
@@ -116,8 +118,7 @@ private class DistanceScorer extends Scorer {
         private final DocIdSetIterator disi;
         private final float weight;
 
-        protected DistanceScorer(Weight weight, AbstractLongFieldScript script, int maxDoc, float boost) {
-            super(weight);
+        protected DistanceScorer(AbstractLongFieldScript script, int maxDoc, float boost) {
             this.script = script;
             twoPhase = new TwoPhaseIterator(DocIdSetIterator.all(maxDoc)) {
                 @Override
diff --git a/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java
index 7c8ac4a8cae63..d18098ee7de33 100644
--- a/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQuery.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.script.AbstractLongFieldScript;
@@ -56,8 +57,10 @@ public boolean isCacheable(LeafReaderContext ctx) {
             }
 
             @Override
-            public Scorer scorer(LeafReaderContext context) {
-                return new DistanceScorer(this, scriptContextFunction().apply(context), context.reader().maxDoc(), boost);
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                return new DefaultScorerSupplier(
+                    new DistanceScorer(scriptContextFunction().apply(context), context.reader().maxDoc(), boost)
+                );
             }
 
             @Override
@@ -84,8 +87,7 @@ private class DistanceScorer extends Scorer {
         private final DocIdSetIterator disi;
         private final float weight;
 
-        protected DistanceScorer(Weight weight, AbstractLongFieldScript script, int maxDoc, float boost) {
-            super(weight);
+        protected DistanceScorer(AbstractLongFieldScript script, int maxDoc, float boost) {
             this.script = script;
             twoPhase = new TwoPhaseIterator(DocIdSetIterator.all(maxDoc)) {
                 @Override
diff --git a/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQuery.java
index ab32427ed4ac1..3c5931367370e 100644
--- a/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldRegexpQuery.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.search.runtime;
 
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.StringFieldScript;
@@ -34,7 +35,12 @@ public StringScriptFieldRegexpQuery(
             script,
             leafFactory,
             fieldName,
-            new ByteRunAutomaton(new RegExp(Objects.requireNonNull(pattern), syntaxFlags, matchFlags).toAutomaton(maxDeterminizedStates))
+            new ByteRunAutomaton(
+                Operations.determinize(
+                    new RegExp(Objects.requireNonNull(pattern), syntaxFlags, matchFlags).toAutomaton(),
+                    maxDeterminizedStates
+                )
+            )
         );
         this.pattern = pattern;
         this.syntaxFlags = syntaxFlags;
diff --git a/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldWildcardQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldWildcardQuery.java
index 5bacaf0d36b55..6c1aa6f72c4a1 100644
--- a/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldWildcardQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/runtime/StringScriptFieldWildcardQuery.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.StringFieldScript;
@@ -44,7 +45,7 @@ private static Automaton buildAutomaton(Term term, boolean caseInsensitive) {
         if (caseInsensitive) {
             return AutomatonQueries.toCaseInsensitiveWildcardAutomaton(term);
         }
-        return WildcardQuery.toAutomaton(term);
+        return WildcardQuery.toAutomaton(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/slice/DocIdSliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/DocIdSliceQuery.java
index f4ab7e29e1684..6de888ac8aff4 100644
--- a/server/src/main/java/org/elasticsearch/search/slice/DocIdSliceQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/slice/DocIdSliceQuery.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 
@@ -58,9 +59,10 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
 
         return new ConstantScoreWeight(this, boost) {
             @Override
-            public Scorer scorer(LeafReaderContext context) {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 DocIdSetIterator iterator = createIterator(context, sliceStart, sliceStart + sliceSize);
-                return new ConstantScoreScorer(this, boost, scoreMode, iterator);
+                Scorer scorer = new ConstantScoreScorer(boost, scoreMode, iterator);
+                return new DefaultScorerSupplier(scorer);
             }
 
             private static DocIdSetIterator createIterator(LeafReaderContext context, int sliceStart, int sliceEnd) {
diff --git a/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
index b66ae219ace97..05cf173468fdc 100644
--- a/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
@@ -20,6 +20,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 
@@ -41,7 +42,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
         return new ConstantScoreWeight(this, boost) {
 
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), getField());
                 final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
                 final TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
@@ -66,7 +67,8 @@ public float matchCost() {
                         return 10;
                     }
                 };
-                return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
+                Scorer scorer = new ConstantScoreScorer(score(), scoreMode, twoPhase);
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
index 3927f54461bb8..9aecbfdd84ee6 100644
--- a/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.DocIdSetBuilder;
@@ -49,10 +50,11 @@ public TermsSliceQuery(String field, int id, int max) {
     public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
         return new ConstantScoreWeight(this, boost) {
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 final DocIdSet disi = build(context.reader());
                 final DocIdSetIterator leafIt = disi.iterator();
-                return new ConstantScoreScorer(this, score(), scoreMode, leafIt);
+                Scorer scorer = new ConstantScoreScorer(score(), scoreMode, leafIt);
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index 56871764ab0d4..6640f0f858404 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -427,18 +426,7 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
 
                 fieldName = currentName;
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parser.getRestApiVersion() == RestApiVersion.V_7
-                    && NESTED_FILTER_FIELD.match(currentName, parser.getDeprecationHandler())) {
-                    deprecationLogger.compatibleCritical(
-                        "nested_filter",
-                        "[nested_filter] has been removed in favour of the [nested] parameter"
-                    );
-                    throw new ParsingException(
-                        parser.getTokenLocation(),
-                        "[nested_filter] has been removed in favour of the [nested] parameter",
-                        currentName
-                    );
-                } else if (NESTED_FIELD.match(currentName, parser.getDeprecationHandler())) {
+                if (NESTED_FIELD.match(currentName, parser.getDeprecationHandler())) {
                     nestedSort = NestedSortBuilder.fromXContent(parser);
                 } else {
                     // the json in the format of -> field : { lat : 30, lon : 12 }
@@ -454,18 +442,7 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
                     geoPoints.add(GeoUtils.parseGeoPoint(parser));
                 }
             } else if (token.isValue()) {
-                if (parser.getRestApiVersion() == RestApiVersion.V_7
-                    && NESTED_PATH_FIELD.match(currentName, parser.getDeprecationHandler())) {
-                    deprecationLogger.compatibleCritical(
-                        "nested_path",
-                        "[nested_path] has been removed in favour of the [nested] parameter"
-                    );
-                    throw new ParsingException(
-                        parser.getTokenLocation(),
-                        "[nested_path] has been removed in favour of the [nested] parameter",
-                        currentName
-                    );
-                } else if (ORDER_FIELD.match(currentName, parser.getDeprecationHandler())) {
+                if (ORDER_FIELD.match(currentName, parser.getDeprecationHandler())) {
                     order = SortOrder.fromString(parser.text());
                 } else if (UNIT_FIELD.match(currentName, parser.getDeprecationHandler())) {
                     unit = DistanceUnit.fromString(parser.text());
diff --git a/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
index a6fd4ef90693d..e60e534d6acaa 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
@@ -126,7 +126,6 @@ public void setScorer(Scorable scorer) {
 
                     @Override
                     protected boolean advanceExact(int doc) throws IOException {
-                        assert doc == scorer.docID() : "expected scorer to be on [" + doc + "] but was on [" + scorer.docID() + "]";
                         /* We will never be called by documents that don't match the
                          * query and they'll all have a score, thus `true`. */
                         score = scorer.score();
diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
index 9f943e63ef1e6..0ac3b42dd5b10 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.mapper.NestedObjectMapper;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -50,6 +51,8 @@ public abstract class SortBuilder<T extends SortBuilder<T>>
 
     // parse fields common to more than one SortBuilder
     public static final ParseField ORDER_FIELD = new ParseField("order");
+
+    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
     public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter").withAllDeprecated()
         .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
     public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path").withAllDeprecated()
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
index 8fdc33f38934f..fd6cfeaea639b 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.CollectionTerminatedException;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Weight;
@@ -84,7 +85,7 @@ private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSu
                 LeafCollector leafCollector = null;
                 try {
                     leafCollector = collector.getLeafCollector(context);
-                    scorer.score(leafCollector, context.reader().getLiveDocs());
+                    scorer.score(leafCollector, context.reader().getLiveDocs(), 0, DocIdSetIterator.NO_MORE_DOCS);
                 } catch (CollectionTerminatedException e) {
                     // collection was terminated prematurely
                     // continue with the following leaf
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
index 1366da366b068..ed8197786ba7e 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
@@ -133,7 +133,7 @@ public CandidateSet drawCandidates(CandidateSet set) throws IOException {
                  * because that's what {@link DirectSpellChecker#suggestSimilar} expects
                  * when filtering terms.
                  */
-                int threshold = thresholdTermFrequency(original.termStats.docFreq);
+                int threshold = thresholdTermFrequency(original.termStats.docFreq());
                 if (threshold == Integer.MAX_VALUE) {
                     // the threshold is the max possible frequency so we can skip the search
                     return set;
@@ -226,7 +226,7 @@ public void nextToken() throws IOException {
     }
 
     private static double score(TermStats termStats, double errorScore, long dictionarySize) {
-        return errorScore * (((double) termStats.totalTermFreq + 1) / ((double) dictionarySize + 1));
+        return errorScore * (((double) termStats.totalTermFreq() + 1) / ((double) dictionarySize + 1));
     }
 
     // package protected for test
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/LaplaceScorer.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/LaplaceScorer.java
index a47cd5fe5a84b..0fd3ebcd00865 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/LaplaceScorer.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/LaplaceScorer.java
@@ -36,7 +36,7 @@ protected double scoreUnigram(Candidate word) throws IOException {
     @Override
     protected double scoreBigram(Candidate word, Candidate w_1) throws IOException {
         join(separator, spare, w_1.term, word.term);
-        return (alpha + frequency(spare.get())) / (w_1.termStats.totalTermFreq + alpha * numTerms);
+        return (alpha + frequency(spare.get())) / (w_1.termStats.totalTermFreq() + alpha * numTerms);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolatingScorer.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolatingScorer.java
index fe64a65498776..0d66311303080 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolatingScorer.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolatingScorer.java
@@ -58,7 +58,7 @@ protected double scoreBigram(Candidate word, Candidate w_1) throws IOException {
         if (count < 1) {
             return unigramLambda * scoreUnigram(word);
         }
-        return bigramLambda * (count / (0.5d + w_1.termStats.totalTermFreq)) + unigramLambda * scoreUnigram(word);
+        return bigramLambda * (count / (0.5d + w_1.termStats.totalTermFreq())) + unigramLambda * scoreUnigram(word);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java
index 7257a0d972459..21d1f34b68eee 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java
@@ -70,7 +70,7 @@ public void nextToken() throws IOException {
                 if (posIncAttr.getPositionIncrement() == 0 && typeAttribute.type() == SynonymFilter.TYPE_SYNONYM) {
                     assert currentSet != null;
                     TermStats termStats = generator.termStats(term);
-                    if (termStats.docFreq > 0) {
+                    if (termStats.docFreq() > 0) {
                         currentSet.addOneCandidate(generator.createCandidate(BytesRef.deepCopyOf(term), termStats, realWordLikelihood));
                     }
                 } else {
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
index d893e0986e0d3..270866c14b20a 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
@@ -35,7 +35,7 @@ protected double scoreBigram(Candidate word, Candidate w_1) throws IOException {
         if (count < 1) {
             return discount * scoreUnigram(word);
         }
-        return count / (w_1.termStats.totalTermFreq + 0.00000000001d);
+        return count / (w_1.termStats.totalTermFreq() + 0.00000000001d);
     }
 
     @Override
@@ -50,7 +50,7 @@ protected double scoreTrigram(Candidate w, Candidate w_1, Candidate w_2) throws
         join(separator, spare, w_2.term, w_1.term, w.term);
         long trigramCount = frequency(spare.get());
         if (trigramCount < 1) {
-            return discount * (bigramCount / (w_1.termStats.totalTermFreq + 0.00000000001d));
+            return discount * (bigramCount / (w_1.termStats.totalTermFreq() + 0.00000000001d));
         }
         return trigramCount / (bigramCount + 0.00000000001d);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/DenseVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/DenseVectorQuery.java
index 44bbd0f50951c..31e19b6784757 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/DenseVectorQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/DenseVectorQuery.java
@@ -19,6 +19,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.VectorScorer;
 import org.apache.lucene.search.Weight;
 
@@ -70,12 +71,12 @@ public Explanation explain(LeafReaderContext leafReaderContext, int i) throws IO
         }
 
         @Override
-        public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
-            VectorScorer vectorScorer = vectorScorer(leafReaderContext);
+        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+            VectorScorer vectorScorer = vectorScorer(context);
             if (vectorScorer == null) {
                 return null;
             }
-            return new DenseVectorScorer(this, vectorScorer);
+            return new DefaultScorerSupplier(new DenseVectorScorer(vectorScorer, boost));
         }
 
         @Override
@@ -178,11 +179,10 @@ static class DenseVectorScorer extends Scorer {
         private final DocIdSetIterator iterator;
         private final float boost;
 
-        DenseVectorScorer(DenseVectorWeight weight, VectorScorer vectorScorer) {
-            super(weight);
+        DenseVectorScorer(VectorScorer vectorScorer, float boost) {
             this.vectorScorer = vectorScorer;
             this.iterator = vectorScorer.iterator();
-            this.boost = weight.boost;
+            this.boost = boost;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenByteKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenByteKnnVectorQuery.java
index 413840f2b451b..9f3d83b4da082 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenByteKnnVectorQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenByteKnnVectorQuery.java
@@ -34,7 +34,7 @@ public ESDiversifyingChildrenByteKnnVectorQuery(
     @Override
     protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
         TopDocs topK = kParam == null ? super.mergeLeafResults(perLeafResults) : TopDocs.merge(kParam, perLeafResults);
-        vectorOpsCount = topK.totalHits.value;
+        vectorOpsCount = topK.totalHits.value();
         return topK;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenFloatKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenFloatKnnVectorQuery.java
index 80704a3b552fe..3907bdf89bc6f 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenFloatKnnVectorQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ESDiversifyingChildrenFloatKnnVectorQuery.java
@@ -34,7 +34,7 @@ public ESDiversifyingChildrenFloatKnnVectorQuery(
     @Override
     protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
         TopDocs topK = kParam == null ? super.mergeLeafResults(perLeafResults) : TopDocs.merge(kParam, perLeafResults);
-        vectorOpsCount = topK.totalHits.value;
+        vectorOpsCount = topK.totalHits.value();
         return topK;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java
index 14bb94a366e50..9363f67a7350b 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java
@@ -27,7 +27,7 @@ public ESKnnByteVectorQuery(String field, byte[] target, Integer k, int numCands
     protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
         // if k param is set, we get only top k results from each shard
         TopDocs topK = kParam == null ? super.mergeLeafResults(perLeafResults) : TopDocs.merge(kParam, perLeafResults);
-        vectorOpsCount = topK.totalHits.value;
+        vectorOpsCount = topK.totalHits.value();
         return topK;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java
index 590d8cfbbaba1..be0437af9131d 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java
@@ -27,7 +27,7 @@ public ESKnnFloatVectorQuery(String field, float[] target, Integer k, int numCan
     protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
         // if k param is set, we get only top k results from each shard
         TopDocs topK = kParam == null ? super.mergeLeafResults(perLeafResults) : TopDocs.merge(kParam, perLeafResults);
-        vectorOpsCount = topK.totalHits.value;
+        vectorOpsCount = topK.totalHits.value();
         return topK;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java
index 30a32e9a615de..c8670a8dfeec2 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java
@@ -56,7 +56,7 @@ public ExactKnnQueryBuilder(StreamInput in) throws IOException {
         }
         this.field = in.readString();
         if (in.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
-            || in.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) {
             this.vectorSimilarity = in.readOptionalFloat();
         } else {
             this.vectorSimilarity = null;
@@ -89,7 +89,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         }
         out.writeString(field);
         if (out.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
-            || out.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) {
             out.writeOptionalFloat(vectorSimilarity);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java
index 06fb109d6580e..bb83b8528c6c8 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java
@@ -18,6 +18,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 
 import java.io.IOException;
@@ -88,13 +89,13 @@ public Explanation explain(LeafReaderContext context, int doc) {
             }
 
             @Override
-            public Scorer scorer(LeafReaderContext context) {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 // Segment starts indicate how many docs are in the segment,
                 // upper equalling lower indicates no documents for this segment
                 if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) {
                     return null;
                 }
-                return new Scorer(this) {
+                Scorer scorer = new Scorer() {
                     final int lower = segmentStarts[context.ord];
                     final int upper = segmentStarts[context.ord + 1];
                     int upTo = -1;
@@ -177,6 +178,7 @@ private int currentDocId() {
                     }
 
                 };
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
index 15052fdad3818..f52addefc8b1c 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
@@ -72,7 +72,7 @@ public KnnScoreDocQueryBuilder(StreamInput in) throws IOException {
             this.queryVector = null;
         }
         if (in.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
-            || in.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) {
             this.vectorSimilarity = in.readOptionalFloat();
         } else {
             this.vectorSimilarity = null;
@@ -117,7 +117,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             }
         }
         if (out.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
-            || out.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) {
             out.writeOptionalFloat(vectorSimilarity);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
index a30aa0efc8bb3..8ce8fc07f3acd 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
@@ -261,7 +261,7 @@ public KnnSearchBuilder(StreamInput in) throws IOException {
         }
         this.filterQueries = in.readNamedWriteableCollectionAsList(QueryBuilder.class);
         this.boost = in.readFloat();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.TOP_LEVEL_KNN_SUPPORT_QUERY_NAME)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.queryName = in.readOptionalString();
         } else {
             this.queryName = null;
@@ -501,7 +501,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeNamedWriteableCollection(filterQueries);
         out.writeFloat(boost);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.TOP_LEVEL_KNN_SUPPORT_QUERY_NAME)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeOptionalString(queryName);
         }
         if (out.getTransportVersion().before(TransportVersions.V_8_7_0) && queryVectorBuilder != null) {
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
index 9af1968b55d14..deb7e6bd035b8 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
@@ -192,7 +192,7 @@ private KnnVectorQueryBuilder(
     public KnnVectorQueryBuilder(StreamInput in) throws IOException {
         super(in);
         this.fieldName = in.readString();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.K_FOR_KNN_QUERY_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.k = in.readOptionalVInt();
         } else {
             this.k = null;
@@ -279,7 +279,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             throw new IllegalStateException("missing a rewriteAndFetch?");
         }
         out.writeString(fieldName);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.K_FOR_KNN_QUERY_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeOptionalVInt(k);
         }
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/VectorSimilarityQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/VectorSimilarityQuery.java
index 77f60adc4fcd8..5219778047bcd 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/VectorSimilarityQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/VectorSimilarityQuery.java
@@ -18,6 +18,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.common.lucene.search.function.MinScoreScorer;
 
@@ -142,12 +143,22 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
         }
 
         @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            Scorer innerScorer = in.scorer(context);
-            if (innerScorer == null) {
+        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+            ScorerSupplier inScorerSupplier = in.scorerSupplier(context);
+            if (inScorerSupplier == null) {
                 return null;
             }
-            return new MinScoreScorer(this, innerScorer, docScore, boost);
+            return new ScorerSupplier() {
+                @Override
+                public Scorer get(long leadCost) throws IOException {
+                    return new MinScoreScorer(inScorerSupplier.get(leadCost), docScore, boost);
+                }
+
+                @Override
+                public long cost() {
+                    return inScorerSupplier.cost();
+                }
+            };
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index e901dc28ea541..cf023b0e629c6 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamAlias;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -52,6 +53,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -398,6 +400,8 @@ private void startRestore(
         Map<String, DataStream> dataStreamsToRestore = result.v1();
         Map<String, DataStreamAlias> dataStreamAliasesToRestore = result.v2();
 
+        validateDataStreamTemplatesExistAndWarnIfMissing(dataStreamsToRestore, snapshotInfo, globalMetadata);
+
         // Remove the data streams from the list of requested indices
         requestIndices.removeAll(dataStreamsToRestore.keySet());
 
@@ -510,6 +514,35 @@ private void startRestore(
         );
     }
 
+    private void validateDataStreamTemplatesExistAndWarnIfMissing(
+        Map<String, DataStream> dataStreamsToRestore,
+        SnapshotInfo snapshotInfo,
+        Metadata globalMetadata
+    ) {
+
+        Stream<ComposableIndexTemplate> streams = Stream.concat(
+            clusterService.state().metadata().templatesV2().values().stream(),
+            globalMetadata == null ? Stream.empty() : globalMetadata.templatesV2().values().stream()
+        );
+
+        Set<String> templatePatterns = streams.filter(cit -> cit.getDataStreamTemplate() != null)
+            .flatMap(cit -> cit.indexPatterns().stream())
+            .collect(Collectors.toSet());
+
+        for (String name : dataStreamsToRestore.keySet()) {
+            if (templatePatterns.stream().noneMatch(pattern -> Regex.simpleMatch(pattern, name))) {
+                String warningMessage = format(
+                    "Snapshot [%s] contains data stream [%s] but custer does not have a matching index template. This will cause"
+                        + " rollover to fail until a matching index template is created",
+                    snapshotInfo.snapshotId(),
+                    name
+                );
+                logger.warn(() -> warningMessage);
+                HeaderWarning.addWarning(warningMessage);
+            }
+        }
+    }
+
     @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here
     private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) {
         clusterService.submitUnbatchedStateUpdateTask(source, task);
diff --git a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java
index b14116b3d55ba..c760e8043e262 100644
--- a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java
+++ b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java
@@ -230,7 +230,7 @@ public void getSynonymSetRules(String synonymSetId, ActionListener<PagedResult<S
         client.prepareSearch(SYNONYMS_ALIAS_NAME)
             .setSource(new SearchSourceBuilder().size(0).trackTotalHits(true))
             .execute(listener.delegateFailureAndWrap((searchListener, countResponse) -> {
-                long totalSynonymRules = countResponse.getHits().getTotalHits().value;
+                long totalSynonymRules = countResponse.getHits().getTotalHits().value();
                 if (totalSynonymRules > maxSynonymsSets) {
                     logger.warn(
                         "The number of synonym rules in the synonym set [{}] exceeds the maximum allowed."
@@ -265,7 +265,7 @@ public void getSynonymSetRules(String synonymSetId, int from, int size, ActionLi
             .setPreference(Preference.LOCAL.type())
             .setTrackTotalHits(true)
             .execute(new DelegatingIndexNotFoundActionListener<>(synonymSetId, listener, (searchListener, searchResponse) -> {
-                final long totalSynonymRules = searchResponse.getHits().getTotalHits().value;
+                final long totalSynonymRules = searchResponse.getHits().getTotalHits().value();
                 // If there are no rules, check that the synonym set actually exists to return the proper error
                 if (totalSynonymRules == 0) {
                     checkSynonymSetExists(synonymSetId, searchListener.delegateFailure((existsListener, response) -> {
@@ -383,7 +383,7 @@ public void putSynonymRule(String synonymsSetId, SynonymRule synonymRule, Action
                 .setPreference(Preference.LOCAL.type())
                 .setTrackTotalHits(true)
                 .execute(l1.delegateFailureAndWrap((searchListener, searchResponse) -> {
-                    long synonymsSetSize = searchResponse.getHits().getTotalHits().value;
+                    long synonymsSetSize = searchResponse.getHits().getTotalHits().value();
                     if (synonymsSetSize >= maxSynonymsSets) {
                         listener.onFailure(
                             new IllegalArgumentException("The number of synonym rules in a synonyms set cannot exceed " + maxSynonymsSets)
diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java b/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java
index 6707d77d6a2d0..d49ac1e29bea6 100644
--- a/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java
+++ b/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java
@@ -115,7 +115,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (description != null) {
             builder.field("description", description);
         }
-        builder.timeField("start_time_in_millis", "start_time", startTime);
+        builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTime);
         if (builder.humanReadable()) {
             builder.field("running_time", new TimeValue(runningTimeNanos, TimeUnit.NANOSECONDS).toString());
         }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
index dd14319e332e0..01069609ac742 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
@@ -109,7 +109,8 @@ private void messageReceived(TcpChannel channel, InboundMessage message, long st
 
         TransportResponseHandler<?> responseHandler = null;
         ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext existing = threadContext.stashContext()) {
+        assert threadContext.isDefaultContext();
+        try (var ignored = threadContext.newStoredContext()) {
             // Place the context with the headers from the message
             threadContext.setHeaders(header.getHeaders());
             threadContext.putTransient("_remote_address", remoteAddress);
diff --git a/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java b/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java
index 279208ecf8b3b..c341b6e2a541c 100644
--- a/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java
+++ b/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.network.HandlingTimeTracker;
 import org.elasticsearch.common.recycler.Recycler;
 import org.elasticsearch.common.transport.NetworkExceptionHelper;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -237,7 +236,7 @@ private void internalSend(
         final long messageSize = reference.length();
         TransportLogger.logOutboundMessage(channel, reference);
         // stash thread context so that channel event loop is not polluted by thread context
-        try (ThreadContext.StoredContext existing = threadPool.getThreadContext().stashContext()) {
+        try (var ignored = threadPool.getThreadContext().newEmptyContext()) {
             channel.sendMessage(reference, new ActionListener<>() {
                 @Override
                 public void onResponse(Void v) {
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
index f0cafb956457e..bff194a798c1b 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
@@ -129,9 +129,8 @@ void collectNodes(ActionListener<Function<String, DiscoveryNode>> listener) {
             final ThreadContext threadContext = threadPool.getThreadContext();
             final ContextPreservingActionListener<Function<String, DiscoveryNode>> contextPreservingActionListener =
                 new ContextPreservingActionListener<>(threadContext.newRestorableContext(false), listener);
-            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            try (var ignore = threadContext.newEmptySystemContext()) {
                 // we stash any context here since this is an internal execution and should not leak any existing context information
-                threadContext.markAsSystemContext();
                 Transport.Connection connection = remoteConnectionManager.getAnyRemoteConnection();
 
                 // Use different action to collect nodes information depending on the connection model
diff --git a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java
index 623f86059de70..2c198caf22354 100644
--- a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java
@@ -328,10 +328,8 @@ private void collectRemoteNodes(Iterator<Supplier<DiscoveryNode>> seedNodesSuppl
                     sniffResponseHandler = new ClusterStateSniffResponseHandler(connection, listener, seedNodesSuppliers);
                 }
 
-                try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-                    // we stash any context here since this is an internal execution and should not leak any
-                    // existing context information.
-                    threadContext.markAsSystemContext();
+                try (var ignored = threadContext.newEmptySystemContext()) {
+                    // we stash any context here since this is an internal execution and should not leak any existing context information.
                     transportService.sendRequest(
                         connection,
                         action,
diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 4e85ba2cf479f..33c8081971202 100644
--- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -1,2 +1,3 @@
 org.elasticsearch.index.codec.Elasticsearch814Codec
 org.elasticsearch.index.codec.Elasticsearch816Codec
+org.elasticsearch.index.codec.Elasticsearch900Codec
diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
index 44c752def351e..b0ef5b780e775 100644
--- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
@@ -130,3 +130,4 @@
 8.15.0,8702002
 8.15.1,8702002
 8.15.2,8702003
+8.15.3,8702003
diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt
index b2a4c7992d1fe..f9a8237d63717 100644
--- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt
+++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt
@@ -28,7 +28,6 @@ BOOTSTRAP_CHECK_PKI_REALM                                       bootstrap-checks
 BOOTSTRAP_CHECK_ROLE_MAPPINGS                                   bootstrap-checks-xpack.html#bootstrap-checks-xpack-role-mappings
 BOOTSTRAP_CHECK_TLS                                             bootstrap-checks-xpack.html#bootstrap-checks-tls
 BOOTSTRAP_CHECK_TOKEN_SSL                                       bootstrap-checks-xpack.html#bootstrap-checks-xpack-token-ssl
-BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP                          security-minimal-setup.html
 CONTACT_SUPPORT                                                 troubleshooting.html#troubleshooting-contact-support
 UNASSIGNED_SHARDS                                               red-yellow-cluster-status.html
 EXECUTABLE_JNA_TMPDIR                                           executable-jna-tmpdir.html
@@ -44,3 +43,4 @@ X_OPAQUE_ID                                                     api-conventions.
 FORMING_SINGLE_NODE_CLUSTERS                                    modules-discovery-bootstrap-cluster.html#modules-discovery-bootstrap-cluster-joining
 CIRCUIT_BREAKER_ERRORS                                          circuit-breaker-errors.html
 ALLOCATION_EXPLAIN_NO_COPIES                                    cluster-allocation-explain.html#no-valid-shard-copy
+ALLOCATION_EXPLAIN_MAX_RETRY                                    cluster-allocation-explain.html#maximum-number-of-retries-exceeded
diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
index 971940041f9b1..e3681cc975988 100644
--- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
@@ -130,3 +130,4 @@
 8.15.0,8512000
 8.15.1,8512000
 8.15.2,8512000
+8.15.3,8512000
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 31739850e2d35..2c6be01c851e4 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.action.search.VersionMismatchException;
 import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.client.internal.AbstractClientHeadersTestCase;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
@@ -816,7 +815,7 @@ public void testIds() {
         ids.put(158, PeerRecoveryNotFound.class);
         ids.put(159, NodeHealthCheckFailureException.class);
         ids.put(160, NoSeedNodeLeftException.class);
-        ids.put(161, VersionMismatchException.class);
+        ids.put(161, null);   // was org.elasticsearch.action.search.VersionMismatchException.class
         ids.put(162, ElasticsearchAuthenticationProcessingError.class);
         ids.put(163, RepositoryConflictException.class);
         ids.put(164, VersionConflictException.class);
diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java
index e3ab7463ad941..6c2cc5c1f4cc0 100644
--- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java
+++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java
@@ -211,7 +211,7 @@ public void testDenseTransportVersions() {
         Set<Integer> missingVersions = new TreeSet<>();
         TransportVersion previous = null;
         for (var tv : TransportVersions.getAllVersions()) {
-            if (tv.before(TransportVersions.V_8_14_0)) {
+            if (tv.before(TransportVersions.V_8_15_2)) {
                 continue;
             }
             if (previous == null) {
diff --git a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
index a9cc6c07e68c8..c5d995710bcec 100644
--- a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -114,13 +114,6 @@ public void testTypeWhenCompatible() throws IOException {
         ) {
             // DocWriteResponse is abstract so we have to sneak a subclass in here to test it.
         };
-        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) {
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                assertThat(parser.map(), hasEntry(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME));
-            }
-        }
 
         try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) {
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java
index e7661bf3848f5..f37b1d1b41712 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java
@@ -23,7 +23,7 @@
 public class NodesStatsRequestParametersTests extends ESTestCase {
 
     public void testReadWriteMetricSet() {
-        for (var version : List.of(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS, TransportVersions.NODES_STATS_ENUM_SET)) {
+        for (var version : List.of(TransportVersions.V_8_15_0, TransportVersions.NODES_STATS_ENUM_SET)) {
             var randSet = randomSubsetOf(Metric.ALL);
             var metricsOut = randSet.isEmpty() ? EnumSet.noneOf(Metric.class) : EnumSet.copyOf(randSet);
             try {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
index 67e5f30f023c9..b59cc13a20ff2 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -323,21 +322,6 @@ private void assertXContent(
 
         AbstractChunkedSerializingTestCase.assertChunkCount(response, params, o -> expectedChunks[0]);
         assertCriticalWarnings(criticalDeprecationWarnings);
-
-        // check the v7 API too
-        AbstractChunkedSerializingTestCase.assertChunkCount(new ChunkedToXContent() {
-            @Override
-            public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
-                return response.toXContentChunkedV7(outerParams);
-            }
-
-            @Override
-            public boolean isFragment() {
-                return response.isFragment();
-            }
-        }, params, o -> expectedChunks[0]++);
-        // the v7 API should not emit any deprecation warnings
-        assertCriticalWarnings();
     }
 
     private static ClusterRerouteResponse createClusterRerouteResponse(ClusterState clusterState) {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
index 733ef1c2d5935..9957acee36b9d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
@@ -50,7 +50,7 @@ public void testBwcSerialization() throws Exception {
                     in.setTransportVersion(out.getTransportVersion());
                     assertEquals(request.getParentTask(), TaskId.readFromStream(in));
                     assertEquals(request.masterNodeTimeout(), in.readTimeValue());
-                    if (in.getTransportVersion().onOrAfter(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS)) {
+                    if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                         assertEquals(request.masterTerm(), in.readVLong());
                     }
                     assertEquals(request.ackTimeout(), in.readTimeValue());
@@ -79,7 +79,7 @@ public void testBwcSerialization() throws Exception {
                 out.setTransportVersion(version);
                 sample.getParentTask().writeTo(out);
                 out.writeTimeValue(sample.masterNodeTimeout());
-                if (out.getTransportVersion().onOrAfter(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                     out.writeVLong(sample.masterTerm());
                 }
                 out.writeTimeValue(sample.ackTimeout());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
index 65464c7f14a5c..bf4a28b9c60b2 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
@@ -12,8 +12,8 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -53,6 +53,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.suggest.document.Completion912PostingsFormat;
 import org.apache.lucene.search.suggest.document.CompletionPostingsFormat;
@@ -327,7 +328,7 @@ public void testTriangle() throws Exception {
     public void testCompletionField() throws Exception {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(false)
-            .setCodec(new Lucene912Codec(Lucene912Codec.Mode.BEST_SPEED) {
+            .setCodec(new Lucene100Codec(Lucene100Codec.Mode.BEST_SPEED) {
                 @Override
                 public PostingsFormat getPostingsFormatForField(String field) {
                     if (field.startsWith("suggest_")) {
@@ -414,25 +415,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
     enum CodecMode {
         BEST_SPEED {
             @Override
-            Lucene912Codec.Mode mode() {
-                return Lucene912Codec.Mode.BEST_SPEED;
+            Lucene100Codec.Mode mode() {
+                return Lucene100Codec.Mode.BEST_SPEED;
             }
         },
 
         BEST_COMPRESSION {
             @Override
-            Lucene912Codec.Mode mode() {
-                return Lucene912Codec.Mode.BEST_COMPRESSION;
+            Lucene100Codec.Mode mode() {
+                return Lucene100Codec.Mode.BEST_COMPRESSION;
             }
         };
 
-        abstract Lucene912Codec.Mode mode();
+        abstract Lucene100Codec.Mode mode();
     }
 
     static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(randomBoolean())
-            .setCodec(new Lucene912Codec(codecMode.mode()));
+            .setCodec(new Lucene100Codec(codecMode.mode()));
         try (IndexWriter writer = new IndexWriter(directory, config)) {
             for (int i = 0; i < numDocs; i++) {
                 final Document doc = new Document();
@@ -640,7 +641,7 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire
         try (DirectoryReader reader = DirectoryReader.open(source)) {
             IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
                 .setUseCompoundFile(randomBoolean())
-                .setCodec(new Lucene912Codec(mode.mode()) {
+                .setCodec(new Lucene100Codec(mode.mode()) {
                     @Override
                     public PostingsFormat getPostingsFormatForField(String field) {
                         return new ES812PostingsFormat();
@@ -687,7 +688,7 @@ static void collectPerFieldStats(SegmentReader reader, IndexDiskUsageStats stats
         final String[] files;
         final Directory directory;
         if (sis.getUseCompoundFile()) {
-            directory = sis.getCodec().compoundFormat().getCompoundReader(reader.directory(), sis, IOContext.READ);
+            directory = sis.getCodec().compoundFormat().getCompoundReader(reader.directory(), sis, IOContext.DEFAULT);
             files = directory.listAll();
         } else {
             directory = reader.directory();
@@ -785,14 +786,15 @@ private static class RandomMatchQuery extends Query {
         public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
             return new ConstantScoreWeight(this, 1.0f) {
                 @Override
-                public Scorer scorer(LeafReaderContext context) {
+                public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                     final FixedBitSet bits = new FixedBitSet(context.reader().maxDoc());
                     for (int i = 0; i < bits.length(); i++) {
                         if (randomBoolean()) {
                             bits.set(i);
                         }
                     }
-                    return new ConstantScoreScorer(this, 1.0f, ScoreMode.COMPLETE_NO_SCORES, new BitSetIterator(bits, bits.length()));
+                    Scorer scorer = new ConstantScoreScorer(1.0f, ScoreMode.COMPLETE_NO_SCORES, new BitSetIterator(bits, bits.length()));
+                    return new DefaultScorerSupplier(scorer);
                 }
 
                 @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
index 4cfd9b66306ad..3bbc03a333438 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
@@ -82,9 +82,6 @@ public void testIndicesOptions() {
         );
         assertThat(getIndexRequest.indicesOptions().wildcardOptions(), equalTo(IndicesOptions.strictExpandOpen().wildcardOptions()));
         assertThat(getIndexRequest.indicesOptions().gatekeeperOptions(), equalTo(IndicesOptions.strictExpandOpen().gatekeeperOptions()));
-        assertThat(
-            getIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true).build())
-        );
+        assertThat(getIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ALL_APPLICABLE));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
index 3f0a6b40ce60a..7640c1e7af308 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
@@ -17,8 +17,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -63,31 +61,8 @@ public void testToXContentIncludesType() throws Exception {
         FieldMappingMetadata fieldMappingMetadata = new FieldMappingMetadata("my field", new BytesArray("{}"));
         mappings.put("index", Collections.singletonMap("field", fieldMappingMetadata));
         GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings);
-        ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "true"));
+        ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("include_type_name", "true"));
 
-        // v7 with include_type_name attaches _doc
-        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) {
-            response.toXContent(builder, params);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                @SuppressWarnings("unchecked")
-                Map<String, Map<String, Map<String, Object>>> index = (Map<String, Map<String, Map<String, Object>>>) parser.map()
-                    .get("index");
-                assertThat(index.get("mappings"), hasKey(MapperService.SINGLE_MAPPING_NAME));
-                assertThat(index.get("mappings").get(MapperService.SINGLE_MAPPING_NAME), hasKey("field"));
-            }
-        }
-
-        // v7 with no include_type_name do not attach _doc
-        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) {
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                @SuppressWarnings("unchecked")
-                Map<String, Map<String, Object>> index = (Map<String, Map<String, Object>>) parser.map().get("index");
-                assertThat(index.get("mappings"), hasKey("field"));
-            }
-        }
         // v8 does not have _doc, even when include_type_name is present
         // (although this throws unconsumed parameter exception in RestGetFieldMappingsAction)
         try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
index 848e46f2b3366..1a30fae1ebc00 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.action.support.IndicesOptions.FailureStoreOptions;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasAction;
@@ -747,14 +747,14 @@ public void testValidation() throws Exception {
         final String defaultRolloverIndexName;
         final boolean useDataStream = randomBoolean();
         final Metadata.Builder builder = Metadata.builder();
-        var failureStoreOptions = FailureStoreOptions.DEFAULT;
+        var defaultSelectorOptions = IndicesOptions.SelectorOptions.DEFAULT;
         if (useDataStream) {
             DataStream dataStream = DataStreamTestHelper.randomInstance()
                 // ensure no replicate data stream
                 .promoteDataStream();
             rolloverTarget = dataStream.getName();
             if (dataStream.isFailureStoreEnabled() && randomBoolean()) {
-                failureStoreOptions = new FailureStoreOptions(false, true);
+                defaultSelectorOptions = IndicesOptions.SelectorOptions.FAILURES;
                 sourceIndexName = dataStream.getFailureStoreWriteIndex().getName();
                 defaultRolloverIndexName = DataStream.getDefaultFailureStoreName(
                     dataStream.getName(),
@@ -815,7 +815,7 @@ public void testValidation() throws Exception {
             true,
             null,
             null,
-            failureStoreOptions.includeFailureIndices()
+            IndicesOptions.SelectorOptions.FAILURES.equals(defaultSelectorOptions)
         );
 
         newIndexName = newIndexName == null ? defaultRolloverIndexName : newIndexName;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
index a7fa81eb24a57..f0190790ba001 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
@@ -66,7 +66,7 @@ public void testConditionsParsing() throws Exception {
             .endObject()
             .endObject();
         try (var parser = createParser(builder)) {
-            request.fromXContent(false, parser);
+            request.fromXContent(parser);
         }
         Map<String, Condition<?>> conditions = request.getConditions().getConditions();
         assertThat(conditions.size(), equalTo(10));
@@ -120,7 +120,7 @@ public void testParsingWithIndexSettings() throws Exception {
             .endObject()
             .endObject();
         try (var parser = createParser(builder)) {
-            request.fromXContent(false, parser);
+            request.fromXContent(parser);
         }
         Map<String, Condition<?>> conditions = request.getConditions().getConditions();
         assertThat(conditions.size(), equalTo(3));
@@ -143,7 +143,7 @@ public void testTypelessMappingParsing() throws Exception {
             .endObject();
 
         try (var parser = createParser(builder)) {
-            request.fromXContent(false, parser);
+            request.fromXContent(parser);
         }
         CreateIndexRequest createIndexRequest = request.getCreateIndexRequest();
         String mapping = createIndexRequest.mappings();
@@ -175,9 +175,7 @@ public void testSerialize() throws Exception {
         );
         originalRequest.lazy(randomBoolean());
         originalRequest.setIndicesOptions(
-            IndicesOptions.builder(originalRequest.indicesOptions())
-                .failureStoreOptions(new IndicesOptions.FailureStoreOptions(randomBoolean(), randomBoolean()))
-                .build()
+            IndicesOptions.builder(originalRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE).build()
         );
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
@@ -188,10 +186,7 @@ public void testSerialize() throws Exception {
                 assertThat(cloneRequest.getNewIndexName(), equalTo(originalRequest.getNewIndexName()));
                 assertThat(cloneRequest.getRolloverTarget(), equalTo(originalRequest.getRolloverTarget()));
                 assertThat(cloneRequest.isLazy(), equalTo(originalRequest.isLazy()));
-                assertThat(
-                    cloneRequest.indicesOptions().failureStoreOptions(),
-                    equalTo(originalRequest.indicesOptions().failureStoreOptions())
-                );
+                assertThat(cloneRequest.indicesOptions().selectorOptions(), equalTo(originalRequest.indicesOptions().selectorOptions()));
                 for (Map.Entry<String, Condition<?>> entry : cloneRequest.getConditions().getConditions().entrySet()) {
                     Condition<?> condition = originalRequest.getConditions().getConditions().get(entry.getKey());
                     // here we compare the string representation as there is some information loss when serializing
@@ -216,7 +211,7 @@ public void testUnknownFields() throws IOException {
         BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random());
         expectThrows(XContentParseException.class, () -> {
             try (var parser = createParser(xContentType.xContent(), mutated)) {
-                request.fromXContent(false, parser);
+                request.fromXContent(parser);
             }
         });
     }
@@ -261,7 +256,7 @@ public void testValidation() {
             RolloverRequest rolloverRequest = new RolloverRequest("alias-index", "new-index-name");
             rolloverRequest.setIndicesOptions(
                 IndicesOptions.builder(rolloverRequest.indicesOptions())
-                    .failureStoreOptions(new IndicesOptions.FailureStoreOptions(true, true))
+                    .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                     .build()
             );
             ActionRequestValidationException validationException = rolloverRequest.validate();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java
index 8f0ff82beab4b..95446149f026b 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
@@ -48,7 +49,9 @@ public void testSettingsProviderIsOverridden() throws Exception {
                             matchingTemplate,
                             ComposableIndexTemplate.builder()
                                 .indexPatterns(List.of("test_index*"))
-                                .template(new Template(Settings.builder().put("test-setting", 1).build(), null, null))
+                                .template(
+                                    new Template(Settings.builder().put("test-setting", 1).put("test-setting-2", 2).build(), null, null)
+                                )
                                 .build()
                         )
                     )
@@ -69,7 +72,7 @@ public void testSettingsProviderIsOverridden() throws Exception {
             public Settings getAdditionalIndexSettings(
                 String indexName,
                 String dataStreamName,
-                boolean timeSeries,
+                IndexMode templateIndexMode,
                 Metadata metadata,
                 Instant resolvedAt,
                 Settings allSettings,
@@ -77,6 +80,24 @@ public Settings getAdditionalIndexSettings(
             ) {
                 return Settings.builder().put("test-setting", 0).build();
             }
+        }, new IndexSettingProvider() {
+            @Override
+            public Settings getAdditionalIndexSettings(
+                String indexName,
+                String dataStreamName,
+                IndexMode templateIndexMode,
+                Metadata metadata,
+                Instant resolvedAt,
+                Settings indexTemplateAndCreateRequestSettings,
+                List<CompressedXContent> combinedTemplateMappings
+            ) {
+                return Settings.builder().put("test-setting-2", 10).build();
+            }
+
+            @Override
+            public boolean overrulesTemplateAndRequestSettings() {
+                return true;
+            }
         });
 
         Template resolvedTemplate = TransportSimulateIndexTemplateAction.resolveTemplate(
@@ -91,5 +112,6 @@ public Settings getAdditionalIndexSettings(
         );
 
         assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1));
+        assertThat(resolvedTemplate.settings().getAsInt("test-setting-2", -1), is(10));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
index 788265d353698..bc7fe7a62a1da 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -34,10 +33,8 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
-import java.util.UUID;
 
 import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals;
 import static org.elasticsearch.ElasticsearchExceptionTests.randomExceptions;
@@ -95,28 +92,6 @@ public static void parseInnerToXContent(XContentParser parser, DocWriteResponse.
         }
     }
 
-    public void testBulkItemResponseShouldContainTypeInV7CompatibilityMode() throws IOException {
-        BulkItemResponse bulkItemResponse = BulkItemResponse.success(
-            randomInt(),
-            DocWriteRequest.OpType.INDEX,
-            new IndexResponse(
-                new ShardId(randomAlphaOfLength(8), UUID.randomUUID().toString(), randomInt()),
-                randomAlphaOfLength(4),
-                randomNonNegativeLong(),
-                randomNonNegativeLong(),
-                randomNonNegativeLong(),
-                true
-            )
-        );
-        XContentBuilder xContentBuilder = bulkItemResponse.toXContent(
-            XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7),
-            ToXContent.EMPTY_PARAMS
-        );
-
-        String json = BytesReference.bytes(xContentBuilder).utf8ToString();
-        assertThat(json, containsString("\"_type\":\"_doc\""));
-    }
-
     public void testFailureToString() {
         Failure failure = new Failure("index", "id", new RuntimeException("test"));
         String toString = failure.toString();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java
index c94e4e46c9ee3..1e651791eb18a 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java
@@ -22,32 +22,74 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Map.entry;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class SimulateBulkRequestTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
-        testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions());
-        testSerialization(getTestPipelineSubstitutions(), null, null);
-        testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), null);
-        testSerialization(getTestPipelineSubstitutions(), null, getTestIndexTemplateSubstitutions());
-        testSerialization(null, getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions());
-        testSerialization(null, getTestComponentTemplateSubstitutions(), null);
-        testSerialization(null, null, getTestIndexTemplateSubstitutions());
-        testSerialization(null, null, null);
-        testSerialization(Map.of(), Map.of(), Map.of());
+        testSerialization(
+            getMapOrEmpty(getTestPipelineSubstitutions()),
+            getMapOrEmpty(getTestComponentTemplateSubstitutions()),
+            getMapOrEmpty(getTestIndexTemplateSubstitutions()),
+            getMapOrEmpty(getTestMappingAddition())
+        );
+    }
+
+    private <K, V> Map<K, V> getMapOrEmpty(Map<K, V> map) {
+        if (randomBoolean()) {
+            return map;
+        } else {
+            return Map.of();
+        }
+    }
+
+    public void testNullsNotAllowed() {
+        assertThrows(
+            NullPointerException.class,
+            () -> new SimulateBulkRequest(
+                null,
+                getTestPipelineSubstitutions(),
+                getTestComponentTemplateSubstitutions(),
+                getTestMappingAddition()
+            )
+        );
+        assertThrows(
+            NullPointerException.class,
+            () -> new SimulateBulkRequest(
+                getTestPipelineSubstitutions(),
+                null,
+                getTestComponentTemplateSubstitutions(),
+                getTestMappingAddition()
+            )
+        );
+        assertThrows(
+            NullPointerException.class,
+            () -> new SimulateBulkRequest(getTestPipelineSubstitutions(), getTestPipelineSubstitutions(), null, getTestMappingAddition())
+        );
+        assertThrows(
+            NullPointerException.class,
+            () -> new SimulateBulkRequest(
+                getTestPipelineSubstitutions(),
+                getTestPipelineSubstitutions(),
+                getTestComponentTemplateSubstitutions(),
+                null
+            )
+        );
     }
 
     private void testSerialization(
         Map<String, Map<String, Object>> pipelineSubstitutions,
         Map<String, Map<String, Object>> componentTemplateSubstitutions,
-        Map<String, Map<String, Object>> indexTemplateSubstitutions
+        Map<String, Map<String, Object>> indexTemplateSubstitutions,
+        Map<String, Object> mappingAddition
     ) throws IOException {
         SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(
             pipelineSubstitutions,
             componentTemplateSubstitutions,
-            indexTemplateSubstitutions
+            indexTemplateSubstitutions,
+            mappingAddition
         );
         /*
          * Note: SimulateBulkRequest does not implement equals or hashCode, so we can't test serialization in the usual way for a
@@ -59,7 +101,7 @@ private void testSerialization(
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
     public void testGetComponentTemplateSubstitutions() throws IOException {
-        SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+        SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
         assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of()));
         String substituteComponentTemplatesString = """
               {
@@ -93,7 +135,7 @@ public void testGetComponentTemplateSubstitutions() throws IOException {
             XContentType.JSON
         ).v2();
         Map<String, Map<String, Object>> substituteComponentTemplates = (Map<String, Map<String, Object>>) tempMap;
-        simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates, Map.of());
+        simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates, Map.of(), Map.of());
         Map<String, ComponentTemplate> componentTemplateSubstitutions = simulateBulkRequest.getComponentTemplateSubstitutions();
         assertThat(componentTemplateSubstitutions.size(), equalTo(2));
         assertThat(
@@ -118,7 +160,7 @@ public void testGetComponentTemplateSubstitutions() throws IOException {
     }
 
     public void testGetIndexTemplateSubstitutions() throws IOException {
-        SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of());
+        SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
         assertThat(simulateBulkRequest.getIndexTemplateSubstitutions(), equalTo(Map.of()));
         String substituteIndexTemplatesString = """
               {
@@ -154,7 +196,7 @@ public void testGetIndexTemplateSubstitutions() throws IOException {
             randomBoolean(),
             XContentType.JSON
         ).v2();
-        simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), substituteIndexTemplates);
+        simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), substituteIndexTemplates, Map.of());
         Map<String, ComposableIndexTemplate> indexTemplateSubstitutions = simulateBulkRequest.getIndexTemplateSubstitutions();
         assertThat(indexTemplateSubstitutions.size(), equalTo(2));
         assertThat(
@@ -179,7 +221,8 @@ public void testShallowClone() throws IOException {
         SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(
             getTestPipelineSubstitutions(),
             getTestComponentTemplateSubstitutions(),
-            getTestIndexTemplateSubstitutions()
+            getTestIndexTemplateSubstitutions(),
+            getTestMappingAddition()
         );
         simulateBulkRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values()));
         simulateBulkRequest.waitForActiveShards(randomIntBetween(1, 10));
@@ -204,7 +247,6 @@ public void testShallowClone() throws IOException {
         assertThat(shallowCopy.routing(), equalTo(simulateBulkRequest.routing()));
         assertThat(shallowCopy.requireAlias(), equalTo(simulateBulkRequest.requireAlias()));
         assertThat(shallowCopy.requireDataStream(), equalTo(simulateBulkRequest.requireDataStream()));
-
     }
 
     private static Map<String, Map<String, Object>> getTestPipelineSubstitutions() {
@@ -248,4 +290,22 @@ private static Map<String, Map<String, Object>> getTestIndexTemplateSubstitution
             Map.of("template", Map.of("index_patterns", List.of("foo*", "bar*"), "mappings", Map.of(), "settings", Map.of()))
         );
     }
+
+    private static Map<String, Object> getTestMappingAddition() {
+        return Map.ofEntries(
+            entry(
+                "_doc",
+                Map.ofEntries(
+                    entry("dynamic", "strict"),
+                    entry(
+                        "properties",
+                        Map.ofEntries(
+                            entry("foo", Map.ofEntries(entry("type", "keyword"))),
+                            entry("bar", Map.ofEntries(entry("type", "boolean")))
+                        )
+                    )
+                )
+            )
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java
index 71bc31334920e..63d308e1579f3 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java
@@ -135,7 +135,7 @@ public void tearDown() throws Exception {
 
     public void testIndexData() throws IOException {
         Task task = mock(Task.class); // unused
-        BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null);
+        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
         int bulkItemCount = randomIntBetween(0, 200);
         for (int i = 0; i < bulkItemCount; i++) {
             Map<String, ?> source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5));
@@ -218,7 +218,11 @@ public void testIndexDataWithValidation() throws IOException {
          * (7) An indexing request to a nonexistent index that matches no templates
          */
         Task task = mock(Task.class); // unused
-        BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null);
+        /*
+         * Here we only add a mapping_addition because if there is no mapping at all TransportSimulateBulkAction skips mapping validation
+         * altogether, and we need it to run for this test to pass.
+         */
+        BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of("_doc", Map.of("dynamic", "strict")));
         int bulkItemCount = randomIntBetween(0, 200);
         Map<String, IndexMetadata> indicesMap = new HashMap<>();
         Map<String, IndexTemplateMetadata> v1Templates = new HashMap<>();
diff --git a/server/src/test/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumerTests.java b/server/src/test/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumerTests.java
index b7919878f9081..681d9d000beef 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumerTests.java
@@ -79,8 +79,8 @@ public void testNullShardResultHandling() throws Exception {
                 queryPhaseResultConsumer.consumeResult(querySearchResult, nextCounter::incrementAndGet);
             }
             var reducePhase = queryPhaseResultConsumer.reduce();
-            assertEquals(0, reducePhase.totalHits().value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, reducePhase.totalHits().relation);
+            assertEquals(0, reducePhase.totalHits().value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, reducePhase.totalHits().relation());
             assertFalse(reducePhase.isEmptyResult());
             assertEquals(10, nextCounter.get());
         }
@@ -94,8 +94,8 @@ public void testEmptyResults() throws Exception {
             )
         ) {
             var reducePhase = queryPhaseResultConsumer.reduce();
-            assertEquals(0, reducePhase.totalHits().value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, reducePhase.totalHits().relation);
+            assertEquals(0, reducePhase.totalHits().value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, reducePhase.totalHits().relation());
             assertTrue(reducePhase.isEmptyResult());
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index 90174a89209b8..99401e8a8d40a 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -151,11 +151,11 @@ public void run() throws IOException {
             assertNotNull(responseRef.get());
             assertNotNull(responseRef.get().get(0));
             assertNull(responseRef.get().get(0).fetchResult());
-            assertEquals(1, responseRef.get().get(0).queryResult().topDocs().topDocs.totalHits.value);
+            assertEquals(1, responseRef.get().get(0).queryResult().topDocs().topDocs.totalHits.value());
             assertEquals(42, responseRef.get().get(0).queryResult().topDocs().topDocs.scoreDocs[0].doc);
             assertNotNull(responseRef.get().get(1));
             assertNull(responseRef.get().get(1).fetchResult());
-            assertEquals(1, responseRef.get().get(1).queryResult().topDocs().topDocs.totalHits.value);
+            assertEquals(1, responseRef.get().get(1).queryResult().topDocs().topDocs.totalHits.value());
             assertEquals(84, responseRef.get().get(1).queryResult().topDocs().topDocs.scoreDocs[0].doc);
             assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
             assertEquals(2, mockSearchPhaseContext.numSuccess.get());
@@ -236,7 +236,7 @@ public void run() throws IOException {
             assertNotNull(responseRef.get());
             assertNotNull(responseRef.get().get(0));
             assertNull(responseRef.get().get(0).fetchResult());
-            assertEquals(1, responseRef.get().get(0).queryResult().topDocs().topDocs.totalHits.value);
+            assertEquals(1, responseRef.get().get(0).queryResult().topDocs().topDocs.totalHits.value());
             assertEquals(42, responseRef.get().get(0).queryResult().topDocs().topDocs.scoreDocs[0].doc);
             assertNull(responseRef.get().get(1));
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 31ef57482cab1..09dd7821cd123 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -116,7 +116,7 @@ public void testShortcutQueryAndFetchOptimization() throws Exception {
             mockSearchPhaseContext.assertNoFailure();
             SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
             assertNotNull(searchResponse);
-            assertEquals(numHits, searchResponse.getHits().getTotalHits().value);
+            assertEquals(numHits, searchResponse.getHits().getTotalHits().value());
             if (numHits != 0) {
                 assertEquals(42, searchResponse.getHits().getAt(0).docId());
             }
@@ -244,7 +244,7 @@ public void sendExecuteFetch(
             mockSearchPhaseContext.assertNoFailure();
             SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
             assertNotNull(searchResponse);
-            assertEquals(2, searchResponse.getHits().getTotalHits().value);
+            assertEquals(2, searchResponse.getHits().getTotalHits().value());
             assertEquals(84, searchResponse.getHits().getAt(0).docId());
             assertEquals(42, searchResponse.getHits().getAt(1).docId());
             assertEquals(0, searchResponse.getFailedShards());
@@ -353,7 +353,7 @@ public void sendExecuteFetch(
             mockSearchPhaseContext.assertNoFailure();
             SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
             assertNotNull(searchResponse);
-            assertEquals(2, searchResponse.getHits().getTotalHits().value);
+            assertEquals(2, searchResponse.getHits().getTotalHits().value());
             assertEquals(84, searchResponse.getHits().getAt(0).docId());
             assertEquals(1, searchResponse.getFailedShards());
             assertEquals(1, searchResponse.getSuccessfulShards());
@@ -468,7 +468,7 @@ public void run() {
             mockSearchPhaseContext.assertNoFailure();
             SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
             assertNotNull(searchResponse);
-            assertEquals(numHits, searchResponse.getHits().getTotalHits().value);
+            assertEquals(numHits, searchResponse.getHits().getTotalHits().value());
             assertEquals(Math.min(numHits, resultSetSize), searchResponse.getHits().getHits().length);
             SearchHit[] hits = searchResponse.getHits().getHits();
             for (int i = 0; i < hits.length; i++) {
@@ -703,7 +703,7 @@ public void sendExecuteFetch(
             mockSearchPhaseContext.assertNoFailure();
             SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
             assertNotNull(searchResponse);
-            assertEquals(2, searchResponse.getHits().getTotalHits().value);
+            assertEquals(2, searchResponse.getHits().getTotalHits().value());
             assertEquals(1, searchResponse.getHits().getHits().length);
             assertEquals(84, searchResponse.getHits().getAt(0).docId());
             assertEquals(0, searchResponse.getFailedShards());
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 857402d1baaac..9a507977c0123 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -297,8 +297,8 @@ public void testMerge() {
                     if (trackTotalHits == SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                         assertNull(mergedResponse.hits.getTotalHits());
                     } else {
-                        assertThat(mergedResponse.hits.getTotalHits().value, equalTo(0L));
-                        assertEquals(mergedResponse.hits.getTotalHits().relation, Relation.EQUAL_TO);
+                        assertThat(mergedResponse.hits.getTotalHits().value(), equalTo(0L));
+                        assertEquals(mergedResponse.hits.getTotalHits().relation(), Relation.EQUAL_TO);
                     }
                     for (SearchHit hit : mergedResponse.hits().getHits()) {
                         SearchPhaseResult searchPhaseResult = fetchResults.get(hit.getShard().getShardId().id());
@@ -415,8 +415,8 @@ protected boolean lessThan(RankDoc a, RankDoc b) {
                     if (trackTotalHits == SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                         assertNull(mergedResponse.hits.getTotalHits());
                     } else {
-                        assertThat(mergedResponse.hits.getTotalHits().value, equalTo(0L));
-                        assertEquals(mergedResponse.hits.getTotalHits().relation, Relation.EQUAL_TO);
+                        assertThat(mergedResponse.hits.getTotalHits().value(), equalTo(0L));
+                        assertEquals(mergedResponse.hits.getTotalHits().relation(), Relation.EQUAL_TO);
                     }
                     int rank = 1;
                     for (SearchHit hit : mergedResponse.hits().getHits()) {
@@ -522,8 +522,8 @@ private static int getTotalQueryHits(AtomicArray<SearchPhaseResult> results) {
         int resultCount = 0;
         for (SearchPhaseResult shardResult : results.asList()) {
             TopDocs topDocs = shardResult.queryResult().topDocs().topDocs;
-            assert topDocs.totalHits.relation == Relation.EQUAL_TO;
-            resultCount += (int) topDocs.totalHits.value;
+            assert topDocs.totalHits.relation() == Relation.EQUAL_TO;
+            resultCount += (int) topDocs.totalHits.value();
         }
         return resultCount;
     }
@@ -784,7 +784,7 @@ public void testConsumerConcurrently() throws Exception {
             assertEquals(max.get(), internalMax.value(), 0.0D);
             assertEquals(1, reduce.sortedTopDocs().scoreDocs().length);
             assertEquals(max.get(), reduce.maxScore(), 0.0f);
-            assertEquals(expectedNumResults, reduce.totalHits().value);
+            assertEquals(expectedNumResults, reduce.totalHits().value());
             assertEquals(max.get(), reduce.sortedTopDocs().scoreDocs()[0].score, 0.0f);
             assertFalse(reduce.sortedTopDocs().isSortedByField());
             assertNull(reduce.sortedTopDocs().sortFields());
@@ -844,7 +844,7 @@ public void testConsumerOnlyAggs() throws Exception {
             assertEquals(max.get(), internalMax.value(), 0.0D);
             assertEquals(0, reduce.sortedTopDocs().scoreDocs().length);
             assertEquals(max.get(), reduce.maxScore(), 0.0f);
-            assertEquals(expectedNumResults, reduce.totalHits().value);
+            assertEquals(expectedNumResults, reduce.totalHits().value());
             assertFalse(reduce.sortedTopDocs().isSortedByField());
             assertNull(reduce.sortedTopDocs().sortFields());
             assertNull(reduce.sortedTopDocs().collapseField());
@@ -902,7 +902,7 @@ public void testConsumerOnlyHits() throws Exception {
             assertAggReduction(request);
             assertEquals(1, reduce.sortedTopDocs().scoreDocs().length);
             assertEquals(max.get(), reduce.maxScore(), 0.0f);
-            assertEquals(expectedNumResults, reduce.totalHits().value);
+            assertEquals(expectedNumResults, reduce.totalHits().value());
             assertEquals(max.get(), reduce.sortedTopDocs().scoreDocs()[0].score, 0.0f);
             assertFalse(reduce.sortedTopDocs().isSortedByField());
             assertNull(reduce.sortedTopDocs().sortFields());
@@ -969,7 +969,7 @@ public void testReduceTopNWithFromOffset() throws Exception {
             ScoreDoc[] scoreDocs = reduce.sortedTopDocs().scoreDocs();
             assertEquals(5, scoreDocs.length);
             assertEquals(100.f, reduce.maxScore(), 0.0f);
-            assertEquals(12, reduce.totalHits().value);
+            assertEquals(12, reduce.totalHits().value());
             assertEquals(95.0f, scoreDocs[0].score, 0.0f);
             assertEquals(94.0f, scoreDocs[1].score, 0.0f);
             assertEquals(93.0f, scoreDocs[2].score, 0.0f);
@@ -1022,7 +1022,7 @@ public void testConsumerSortByField() throws Exception {
             SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
             assertAggReduction(request);
             assertEquals(Math.min(expectedNumResults, size), reduce.sortedTopDocs().scoreDocs().length);
-            assertEquals(expectedNumResults, reduce.totalHits().value);
+            assertEquals(expectedNumResults, reduce.totalHits().value());
             assertEquals(max.get(), ((FieldDoc) reduce.sortedTopDocs().scoreDocs()[0]).fields[0]);
             assertTrue(reduce.sortedTopDocs().isSortedByField());
             assertEquals(1, reduce.sortedTopDocs().sortFields().length);
@@ -1079,7 +1079,7 @@ public void testConsumerFieldCollapsing() throws Exception {
             SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
             assertAggReduction(request);
             assertEquals(3, reduce.sortedTopDocs().scoreDocs().length);
-            assertEquals(expectedNumResults, reduce.totalHits().value);
+            assertEquals(expectedNumResults, reduce.totalHits().value());
             assertEquals(a, ((FieldDoc) reduce.sortedTopDocs().scoreDocs()[0]).fields[0]);
             assertEquals(b, ((FieldDoc) reduce.sortedTopDocs().scoreDocs()[1]).fields[0]);
             assertEquals(c, ((FieldDoc) reduce.sortedTopDocs().scoreDocs()[2]).fields[0]);
@@ -1199,7 +1199,7 @@ public void testConsumerSuggestions() throws Exception {
             assertEquals(maxScoreCompletion, reduce.sortedTopDocs().scoreDocs()[0].score, 0f);
             assertEquals(0, reduce.sortedTopDocs().scoreDocs()[0].doc);
             assertNotEquals(-1, reduce.sortedTopDocs().scoreDocs()[0].shardIndex);
-            assertEquals(0, reduce.totalHits().value);
+            assertEquals(0, reduce.totalHits().value());
             assertFalse(reduce.sortedTopDocs().isSortedByField());
             assertNull(reduce.sortedTopDocs().sortFields());
             assertNull(reduce.sortedTopDocs().collapseField());
@@ -1290,7 +1290,7 @@ public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, Interna
                 assertEquals(max.get(), internalMax.value(), 0.0D);
                 assertEquals(1, reduce.sortedTopDocs().scoreDocs().length);
                 assertEquals(max.get(), reduce.maxScore(), 0.0f);
-                assertEquals(expectedNumResults, reduce.totalHits().value);
+                assertEquals(expectedNumResults, reduce.totalHits().value());
                 assertEquals(max.get(), reduce.sortedTopDocs().scoreDocs()[0].score, 0.0f);
                 assertFalse(reduce.sortedTopDocs().isSortedByField());
                 assertNull(reduce.sortedTopDocs().sortFields());
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
index b63c88f623e21..e4284937474c7 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -13,26 +13,18 @@
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
-import org.elasticsearch.cluster.node.VersionInformation;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.lucene.grouping.TopFieldGroups;
 import org.elasticsearch.search.DocValueFormat;
@@ -47,24 +39,17 @@
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalAggregationTestCase;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.transport.Transport;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static java.util.Collections.singletonList;
-import static org.elasticsearch.test.VersionUtils.allVersions;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class SearchQueryThenFetchAsyncActionTests extends ESTestCase {
@@ -245,11 +230,11 @@ public void run() {
             SearchPhaseController.ReducedQueryPhase phase = action.results.reduce();
             assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1));
             if (withScroll) {
-                assertThat(phase.totalHits().value, equalTo((long) numShards));
-                assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+                assertThat(phase.totalHits().value(), equalTo((long) numShards));
+                assertThat(phase.totalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             } else {
-                assertThat(phase.totalHits().value, equalTo(2L));
-                assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+                assertThat(phase.totalHits().value(), equalTo(2L));
+                assertThat(phase.totalHits().relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
             }
             assertThat(phase.sortedTopDocs().scoreDocs().length, equalTo(1));
             assertThat(phase.sortedTopDocs().scoreDocs()[0], instanceOf(FieldDoc.class));
@@ -257,484 +242,4 @@ public void run() {
             assertThat(((FieldDoc) phase.sortedTopDocs().scoreDocs()[0]).fields[0], equalTo(0));
         }
     }
-
-    public void testMinimumVersionSameAsNewVersion() throws Exception {
-        var newVersion = VersionInformation.CURRENT;
-        var oldVersion = new VersionInformation(
-            VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()),
-            IndexVersions.MINIMUM_COMPATIBLE,
-            IndexVersionUtils.randomCompatibleVersion(random())
-        );
-        testMixedVersionsShardsSearch(newVersion, oldVersion, newVersion.nodeVersion());
-    }
-
-    public void testMinimumVersionBetweenNewAndOldVersion() throws Exception {
-        var oldVersion = new VersionInformation(
-            VersionUtils.getFirstVersion(),
-            IndexVersions.MINIMUM_COMPATIBLE,
-            IndexVersionUtils.randomCompatibleVersion(random())
-        );
-
-        var newVersion = new VersionInformation(
-            VersionUtils.maxCompatibleVersion(VersionUtils.getFirstVersion()),
-            IndexVersions.MINIMUM_COMPATIBLE,
-            IndexVersion.current()
-        );
-
-        var minVersion = VersionUtils.randomVersionBetween(
-            random(),
-            allVersions().get(allVersions().indexOf(oldVersion.nodeVersion()) + 1),
-            newVersion.nodeVersion()
-        );
-
-        testMixedVersionsShardsSearch(newVersion, oldVersion, minVersion);
-    }
-
-    private void testMixedVersionsShardsSearch(VersionInformation oldVersion, VersionInformation newVersion, Version minVersion)
-        throws Exception {
-        final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(
-            0,
-            System.nanoTime(),
-            System::nanoTime
-        );
-        int numConcurrent = randomIntBetween(1, 4);
-
-        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
-        DiscoveryNode newVersionNode = DiscoveryNodeUtils.builder("node1").version(newVersion).build();
-        DiscoveryNode oldVersionNode = DiscoveryNodeUtils.builder("node2").version(oldVersion).build();
-        lookup.put("node1", new SearchAsyncActionTests.MockConnection(newVersionNode));
-        lookup.put("node2", new SearchAsyncActionTests.MockConnection(oldVersionNode));
-
-        OriginalIndices idx = new OriginalIndices(new String[] { "idx" }, SearchRequest.DEFAULT_INDICES_OPTIONS);
-        ArrayList<SearchShardIterator> list = new ArrayList<>();
-        ShardRouting routingNewVersionShard = ShardRouting.newUnassigned(
-            new ShardId(new Index("idx", "_na_"), 0),
-            true,
-            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
-            ShardRouting.Role.DEFAULT
-        );
-        routingNewVersionShard = routingNewVersionShard.initialize(newVersionNode.getId(), "p0", 0);
-        routingNewVersionShard.started();
-        list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 0), singletonList(routingNewVersionShard), idx));
-
-        ShardRouting routingOldVersionShard = ShardRouting.newUnassigned(
-            new ShardId(new Index("idx", "_na_"), 1),
-            true,
-            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
-            ShardRouting.Role.DEFAULT
-        );
-        routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p1", 0);
-        routingOldVersionShard.started();
-        list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 1), singletonList(routingOldVersionShard), idx));
-
-        GroupShardsIterator<SearchShardIterator> shardsIter = new GroupShardsIterator<>(list);
-        final SearchRequest searchRequest = new SearchRequest(minVersion);
-        searchRequest.setMaxConcurrentShardRequests(numConcurrent);
-        searchRequest.setBatchedReduceSize(2);
-        searchRequest.source(new SearchSourceBuilder().size(1));
-        searchRequest.allowPartialSearchResults(false);
-
-        SearchTransportService searchTransportService = new SearchTransportService(null, null, null);
-        SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder());
-        SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap());
-        try (
-            QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer(
-                searchRequest,
-                EsExecutors.DIRECT_EXECUTOR_SERVICE,
-                new NoopCircuitBreaker(CircuitBreaker.REQUEST),
-                controller,
-                task::isCancelled,
-                task.getProgressListener(),
-                shardsIter.size(),
-                exc -> {}
-            )
-        ) {
-            final List<Object> responses = new ArrayList<>();
-            SearchQueryThenFetchAsyncAction newSearchAsyncAction = new SearchQueryThenFetchAsyncAction(
-                logger,
-                null,
-                searchTransportService,
-                (clusterAlias, node) -> lookup.get(node),
-                Collections.singletonMap("_na_", AliasFilter.EMPTY),
-                Collections.emptyMap(),
-                EsExecutors.DIRECT_EXECUTOR_SERVICE,
-                resultConsumer,
-                searchRequest,
-                new ActionListener<>() {
-                    @Override
-                    public void onFailure(Exception e) {
-                        responses.add(e);
-                    }
-
-                    public void onResponse(SearchResponse response) {
-                        responses.add(response);
-                    }
-
-                ;
-                },
-                shardsIter,
-                timeProvider,
-                new ClusterState.Builder(new ClusterName("test")).build(),
-                task,
-                SearchResponse.Clusters.EMPTY,
-                null
-            );
-
-            newSearchAsyncAction.start();
-            assertThat(responses, hasSize(1));
-            assertThat(responses.get(0), instanceOf(SearchPhaseExecutionException.class));
-            SearchPhaseExecutionException e = (SearchPhaseExecutionException) responses.get(0);
-            assertThat(e.getCause(), instanceOf(VersionMismatchException.class));
-            assertThat(
-                e.getCause().getMessage(),
-                equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]")
-            );
-        }
-    }
-
-    public void testMinimumVersionSameAsOldVersion() throws Exception {
-        var newVersion = VersionInformation.CURRENT;
-        var oldVersion = new VersionInformation(
-            VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()),
-            IndexVersions.MINIMUM_COMPATIBLE,
-            IndexVersionUtils.randomCompatibleVersion(random())
-        );
-        Version minVersion = oldVersion.nodeVersion();
-
-        final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(
-            0,
-            System.nanoTime(),
-            System::nanoTime
-        );
-        AtomicInteger successfulOps = new AtomicInteger();
-
-        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
-        DiscoveryNode newVersionNode = DiscoveryNodeUtils.builder("node1").version(newVersion).build();
-        DiscoveryNode oldVersionNode = DiscoveryNodeUtils.builder("node2").version(oldVersion).build();
-        lookup.put("node1", new SearchAsyncActionTests.MockConnection(newVersionNode));
-        lookup.put("node2", new SearchAsyncActionTests.MockConnection(oldVersionNode));
-
-        OriginalIndices idx = new OriginalIndices(new String[] { "idx" }, SearchRequest.DEFAULT_INDICES_OPTIONS);
-        ArrayList<SearchShardIterator> list = new ArrayList<>();
-        ShardRouting routingNewVersionShard = ShardRouting.newUnassigned(
-            new ShardId(new Index("idx", "_na_"), 0),
-            true,
-            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
-            ShardRouting.Role.DEFAULT
-        );
-        routingNewVersionShard = routingNewVersionShard.initialize(newVersionNode.getId(), "p0", 0);
-        routingNewVersionShard.started();
-        list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 0), singletonList(routingNewVersionShard), idx));
-
-        ShardRouting routingOldVersionShard = ShardRouting.newUnassigned(
-            new ShardId(new Index("idx", "_na_"), 1),
-            true,
-            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
-            ShardRouting.Role.DEFAULT
-        );
-        routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p1", 0);
-        routingOldVersionShard.started();
-        list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 1), singletonList(routingOldVersionShard), idx));
-
-        GroupShardsIterator<SearchShardIterator> shardsIter = new GroupShardsIterator<>(list);
-        final SearchRequest searchRequest = new SearchRequest(minVersion);
-        searchRequest.allowPartialSearchResults(false);
-        searchRequest.source(new SearchSourceBuilder().size(1).sort(SortBuilders.fieldSort("timestamp")));
-
-        SearchTransportService searchTransportService = new SearchTransportService(null, null, null) {
-            @Override
-            public void sendExecuteQuery(
-                Transport.Connection connection,
-                ShardSearchRequest request,
-                SearchTask task,
-                ActionListener<SearchPhaseResult> listener
-            ) {
-                int shardId = request.shardId().id();
-                QuerySearchResult queryResult = new QuerySearchResult(
-                    new ShardSearchContextId("N/A", 123),
-                    new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null),
-                    null
-                );
-                try {
-                    SortField sortField = new SortField("timestamp", SortField.Type.LONG);
-                    if (shardId == 0) {
-                        queryResult.topDocs(
-                            new TopDocsAndMaxScore(
-                                new TopFieldDocs(
-                                    new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
-                                    new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) },
-                                    new SortField[] { sortField }
-                                ),
-                                Float.NaN
-                            ),
-                            new DocValueFormat[] { DocValueFormat.RAW }
-                        );
-                    } else if (shardId == 1) {
-                        queryResult.topDocs(
-                            new TopDocsAndMaxScore(
-                                new TopFieldDocs(
-                                    new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
-                                    new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) },
-                                    new SortField[] { sortField }
-                                ),
-                                Float.NaN
-                            ),
-                            new DocValueFormat[] { DocValueFormat.RAW }
-                        );
-                    }
-                    queryResult.from(0);
-                    queryResult.size(1);
-                    successfulOps.incrementAndGet();
-                    queryResult.incRef();
-                    new Thread(() -> ActionListener.respondAndRelease(listener, queryResult)).start();
-                } finally {
-                    queryResult.decRef();
-                }
-            }
-        };
-        SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder());
-        SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap());
-        try (
-            QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer(
-                searchRequest,
-                EsExecutors.DIRECT_EXECUTOR_SERVICE,
-                new NoopCircuitBreaker(CircuitBreaker.REQUEST),
-                controller,
-                task::isCancelled,
-                task.getProgressListener(),
-                shardsIter.size(),
-                exc -> {}
-            )
-        ) {
-            CountDownLatch latch = new CountDownLatch(1);
-            SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction(
-                logger,
-                null,
-                searchTransportService,
-                (clusterAlias, node) -> lookup.get(node),
-                Collections.singletonMap("_na_", AliasFilter.EMPTY),
-                Collections.emptyMap(),
-                EsExecutors.DIRECT_EXECUTOR_SERVICE,
-                resultConsumer,
-                searchRequest,
-                null,
-                shardsIter,
-                timeProvider,
-                new ClusterState.Builder(new ClusterName("test")).build(),
-                task,
-                SearchResponse.Clusters.EMPTY,
-                null
-            ) {
-                @Override
-                protected SearchPhase getNextPhase(SearchPhaseResults<SearchPhaseResult> results, SearchPhaseContext context) {
-                    return new SearchPhase("test") {
-                        @Override
-                        public void run() {
-                            latch.countDown();
-                        }
-                    };
-                }
-            };
-
-            action.start();
-            latch.await();
-            assertThat(successfulOps.get(), equalTo(2));
-            SearchPhaseController.ReducedQueryPhase phase = action.results.reduce();
-            assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1));
-            assertThat(phase.totalHits().value, equalTo(2L));
-            assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
-        }
-    }
-
-    public void testMinimumVersionShardDuringPhaseExecution() throws Exception {
-        var newVersion = VersionInformation.CURRENT;
-        var oldVersion = new VersionInformation(
-            VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()),
-            IndexVersions.MINIMUM_COMPATIBLE,
-            IndexVersionUtils.randomCompatibleVersion(random())
-        );
-
-        Version minVersion = newVersion.nodeVersion();
-
-        final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(
-            0,
-            System.nanoTime(),
-            System::nanoTime
-        );
-        AtomicInteger successfulOps = new AtomicInteger();
-
-        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
-        DiscoveryNode newVersionNode1 = DiscoveryNodeUtils.builder("node1").version(newVersion).build();
-        DiscoveryNode newVersionNode2 = DiscoveryNodeUtils.builder("node2").version(newVersion).build();
-        DiscoveryNode oldVersionNode = DiscoveryNodeUtils.builder("node3").version(oldVersion).build();
-        lookup.put("node1", new SearchAsyncActionTests.MockConnection(newVersionNode1));
-        lookup.put("node2", new SearchAsyncActionTests.MockConnection(newVersionNode2));
-        lookup.put("node3", new SearchAsyncActionTests.MockConnection(oldVersionNode));
-
-        OriginalIndices idx = new OriginalIndices(new String[] { "idx" }, SearchRequest.DEFAULT_INDICES_OPTIONS);
-        ArrayList<SearchShardIterator> list = new ArrayList<>();
-        ShardRouting routingNewVersionShard1 = ShardRouting.newUnassigned(
-            new ShardId(new Index("idx", "_na_"), 0),
-            true,
-            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
-            ShardRouting.Role.DEFAULT
-        );
-        routingNewVersionShard1 = routingNewVersionShard1.initialize(newVersionNode1.getId(), "p0", 0);
-        routingNewVersionShard1.started();
-        list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 0), singletonList(routingNewVersionShard1), idx));
-
-        ShardRouting routingNewVersionShard2 = ShardRouting.newUnassigned(
-            new ShardId(new Index("idx", "_na_"), 1),
-            true,
-            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
-            ShardRouting.Role.DEFAULT
-        );
-        routingNewVersionShard2 = routingNewVersionShard2.initialize(newVersionNode2.getId(), "p1", 0);
-        routingNewVersionShard2.started();
-        list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 1), singletonList(routingNewVersionShard2), idx));
-
-        GroupShardsIterator<SearchShardIterator> shardsIter = new GroupShardsIterator<>(list);
-        final SearchRequest searchRequest = new SearchRequest(minVersion);
-        searchRequest.allowPartialSearchResults(false);
-        searchRequest.source(new SearchSourceBuilder().size(1).sort(SortBuilders.fieldSort("timestamp")));
-
-        SearchTransportService searchTransportService = new SearchTransportService(null, null, null) {
-            @Override
-            public void sendExecuteQuery(
-                Transport.Connection connection,
-                ShardSearchRequest request,
-                SearchTask task,
-                ActionListener<SearchPhaseResult> listener
-            ) {
-                int shardId = request.shardId().id();
-                QuerySearchResult queryResult = new QuerySearchResult(
-                    new ShardSearchContextId("N/A", 123),
-                    new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null),
-                    null
-                );
-                try {
-                    SortField sortField = new SortField("timestamp", SortField.Type.LONG);
-                    if (shardId == 0) {
-                        queryResult.topDocs(
-                            new TopDocsAndMaxScore(
-                                new TopFieldDocs(
-                                    new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
-                                    new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) },
-                                    new SortField[] { sortField }
-                                ),
-                                Float.NaN
-                            ),
-                            new DocValueFormat[] { DocValueFormat.RAW }
-                        );
-                    } else if (shardId == 1) {
-                        queryResult.topDocs(
-                            new TopDocsAndMaxScore(
-                                new TopFieldDocs(
-                                    new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
-                                    new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) },
-                                    new SortField[] { sortField }
-                                ),
-                                Float.NaN
-                            ),
-                            new DocValueFormat[] { DocValueFormat.RAW }
-                        );
-                    }
-                    queryResult.from(0);
-                    queryResult.size(1);
-                    successfulOps.incrementAndGet();
-                    queryResult.incRef();
-                    new Thread(() -> ActionListener.respondAndRelease(listener, queryResult)).start();
-                } finally {
-                    queryResult.decRef();
-                }
-            }
-        };
-        SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder());
-        SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap());
-
-        CountDownLatch latch = new CountDownLatch(1);
-        try (
-            QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer(
-                searchRequest,
-                EsExecutors.DIRECT_EXECUTOR_SERVICE,
-                new NoopCircuitBreaker(CircuitBreaker.REQUEST),
-                controller,
-                task::isCancelled,
-                task.getProgressListener(),
-                shardsIter.size(),
-                exc -> {}
-            )
-        ) {
-            SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction(
-                logger,
-                null,
-                searchTransportService,
-                (clusterAlias, node) -> lookup.get(node),
-                Collections.singletonMap("_na_", AliasFilter.EMPTY),
-                Collections.emptyMap(),
-                EsExecutors.DIRECT_EXECUTOR_SERVICE,
-                resultConsumer,
-                searchRequest,
-                null,
-                shardsIter,
-                timeProvider,
-                new ClusterState.Builder(new ClusterName("test")).build(),
-                task,
-                SearchResponse.Clusters.EMPTY,
-                null
-            ) {
-                @Override
-                protected SearchPhase getNextPhase(SearchPhaseResults<SearchPhaseResult> results, SearchPhaseContext context) {
-                    return new SearchPhase("test") {
-                        @Override
-                        public void run() {
-                            latch.countDown();
-                        }
-                    };
-                }
-            };
-            ShardRouting routingOldVersionShard = ShardRouting.newUnassigned(
-                new ShardId(new Index("idx", "_na_"), 2),
-                true,
-                RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
-                ShardRouting.Role.DEFAULT
-            );
-            SearchShardIterator shardIt = new SearchShardIterator(
-                null,
-                new ShardId(new Index("idx", "_na_"), 2),
-                singletonList(routingOldVersionShard),
-                idx
-            );
-            routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p2", 0);
-            routingOldVersionShard.started();
-            action.start();
-            latch.await();
-            assertThat(successfulOps.get(), equalTo(2));
-            SearchPhaseController.ReducedQueryPhase phase = action.results.reduce();
-            assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1));
-            assertThat(phase.totalHits().value, equalTo(2L));
-            assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
-
-            SearchShardTarget searchShardTarget = new SearchShardTarget("node3", shardIt.shardId(), null);
-            SearchActionListener<SearchPhaseResult> listener = new SearchActionListener<SearchPhaseResult>(searchShardTarget, 0) {
-                @Override
-                public void onFailure(Exception e) {}
-
-                @Override
-                protected void innerOnResponse(SearchPhaseResult response) {}
-            };
-            Exception e = expectThrows(
-                VersionMismatchException.class,
-                () -> action.executePhaseOnShard(shardIt, searchShardTarget, listener)
-            );
-            assertThat(e.getMessage(), equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]"));
-        }
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index 23c956e6e52f2..526961d74bf52 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -16,9 +16,12 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ArrayUtils;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
@@ -39,7 +42,6 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -103,6 +105,23 @@ public void testSerialization() throws Exception {
         assertNotSame(deserializedRequest, searchRequest);
     }
 
+    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)  // this can be removed when the affected transport version constants are collapsed
+    public void testSerializationConstants() throws Exception {
+        SearchRequest searchRequest = createSearchRequest();
+
+        // something serialized with previous version to remove, should read correctly with the reversion
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setTransportVersion(TransportVersionUtils.getPreviousVersion(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE));
+            searchRequest.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                in.setTransportVersion(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE);
+                SearchRequest copiedRequest = new SearchRequest(in);
+                assertEquals(copiedRequest, searchRequest);
+                assertEquals(copiedRequest.hashCode(), searchRequest.hashCode());
+            }
+        }
+    }
+
     public void testSerializationMultiKNN() throws Exception {
         SearchRequest searchRequest = createSearchRequest();
         if (searchRequest.source() == null) {
@@ -272,10 +291,23 @@ public void testValidate() throws IOException {
             assertNotNull(validationErrors);
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals(
-                "cannot specify a compound retriever and [allow_partial_search_results]",
+                "cannot specify [test_compound_retriever_builder] and [allow_partial_search_results]",
                 validationErrors.validationErrors().get(0)
             );
         }
+        {
+            // scroll and compound retriever
+            SearchRequest searchRequest = createSearchRequest().source(
+                new SearchSourceBuilder().retriever(new TestCompoundRetrieverBuilder(randomIntBetween(1, 10)))
+            );
+            searchRequest.allowPartialSearchResults(false);
+            searchRequest.scroll(TimeValue.timeValueMinutes(1));
+            searchRequest.requestCache(false);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertEquals(1, validationErrors.validationErrors().size());
+            assertEquals("cannot specify [test_compound_retriever_builder] and [scroll]", validationErrors.validationErrors().get(0));
+        }
         {
             // allow_partial_results and non-compound retriever
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder().retriever(new RetrieverBuilder() {
@@ -438,33 +470,6 @@ public QueryBuilder topDocsQuery() {
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals("using [point in time] is not allowed in a scroll context", validationErrors.validationErrors().get(0));
         }
-        {
-            // Minimum compatible shard node version with ccs_minimize_roundtrips
-            SearchRequest searchRequest;
-            boolean isMinCompatibleShardVersion = randomBoolean();
-            if (isMinCompatibleShardVersion) {
-                searchRequest = new SearchRequest(VersionUtils.randomVersion(random()));
-            } else {
-                searchRequest = new SearchRequest();
-            }
-
-            boolean shouldSetCcsMinimizeRoundtrips = randomBoolean();
-            if (shouldSetCcsMinimizeRoundtrips) {
-                searchRequest.setCcsMinimizeRoundtrips(true);
-            }
-            ActionRequestValidationException validationErrors = searchRequest.validate();
-
-            if (isMinCompatibleShardVersion && shouldSetCcsMinimizeRoundtrips) {
-                assertNotNull(validationErrors);
-                assertEquals(1, validationErrors.validationErrors().size());
-                assertEquals(
-                    "[ccs_minimize_roundtrips] cannot be [true] when setting a minimum compatible shard version",
-                    validationErrors.validationErrors().get(0)
-                );
-            } else {
-                assertNull(validationErrors);
-            }
-        }
         {
             SearchRequest searchRequest = new SearchRequest().source(
                 new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100))
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
index 2b0ed0552e594..51796f404c283 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
@@ -762,11 +762,11 @@ public void testMergeSearchHits() throws InterruptedException {
                 TotalHits totalHits = null;
                 if (trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                     totalHits = new TotalHits(randomLongBetween(0, 1000), totalHitsRelation);
-                    long previousValue = expectedTotalHits == null ? 0 : expectedTotalHits.value;
-                    expectedTotalHits = new TotalHits(Math.min(previousValue + totalHits.value, trackTotalHitsUpTo), totalHitsRelation);
+                    long previousValue = expectedTotalHits == null ? 0 : expectedTotalHits.value();
+                    expectedTotalHits = new TotalHits(Math.min(previousValue + totalHits.value(), trackTotalHitsUpTo), totalHitsRelation);
                 }
 
-                final int numDocs = totalHits == null || totalHits.value >= requestedSize ? requestedSize : (int) totalHits.value;
+                final int numDocs = totalHits == null || totalHits.value() >= requestedSize ? requestedSize : (int) totalHits.value();
                 int scoreFactor = randomIntBetween(1, numResponses);
                 float maxScore = scoreSort ? numDocs * scoreFactor : Float.NaN;
                 SearchHit[] hits = randomSearchHitArray(
@@ -862,8 +862,8 @@ public void testMergeSearchHits() throws InterruptedException {
                     assertNull(searchHits.getTotalHits());
                 } else {
                     assertNotNull(searchHits.getTotalHits());
-                    assertEquals(expectedTotalHits.value, searchHits.getTotalHits().value);
-                    assertSame(expectedTotalHits.relation, searchHits.getTotalHits().relation);
+                    assertEquals(expectedTotalHits.value(), searchHits.getTotalHits().value());
+                    assertSame(expectedTotalHits.relation(), searchHits.getTotalHits().relation());
                 }
                 if (expectedMaxScore == Float.NEGATIVE_INFINITY) {
                     assertTrue(Float.isNaN(searchHits.getMaxScore()));
@@ -910,9 +910,9 @@ public void testMergeNoResponsesAdded() {
                 assertEquals(0, response.getNumReducePhases());
                 assertFalse(response.isTimedOut());
                 assertNotNull(response.getHits().getTotalHits());
-                assertEquals(0, response.getHits().getTotalHits().value);
+                assertEquals(0, response.getHits().getTotalHits().value());
                 assertEquals(0, response.getHits().getHits().length);
-                assertEquals(TotalHits.Relation.EQUAL_TO, response.getHits().getTotalHits().relation);
+                assertEquals(TotalHits.Relation.EQUAL_TO, response.getHits().getTotalHits().relation());
                 assertNull(response.getScrollId());
                 assertSame(InternalAggregations.EMPTY, response.getAggregations());
                 assertNull(response.getSuggest());
@@ -1004,7 +1004,7 @@ public void testMergeEmptySearchHitsWithNonEmpty() {
             assertEquals(2, merger.numResponses());
             SearchResponse mergedResponse = merger.getMergedResponse(clusters);
             try {
-                assertEquals(10, mergedResponse.getHits().getTotalHits().value);
+                assertEquals(10, mergedResponse.getHits().getTotalHits().value());
                 assertEquals(10, mergedResponse.getHits().getHits().length);
                 assertEquals(2, mergedResponse.getTotalShards());
                 assertEquals(2, mergedResponse.getSuccessfulShards());
@@ -1032,8 +1032,8 @@ public void testMergeOnlyEmptyHits() {
                 TotalHits totalHits = null;
                 if (trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                     totalHits = new TotalHits(randomLongBetween(0, 1000), totalHitsRelation);
-                    long previousValue = expectedTotalHits == null ? 0 : expectedTotalHits.value;
-                    expectedTotalHits = new TotalHits(Math.min(previousValue + totalHits.value, trackTotalHitsUpTo), totalHitsRelation);
+                    long previousValue = expectedTotalHits == null ? 0 : expectedTotalHits.value();
+                    expectedTotalHits = new TotalHits(Math.min(previousValue + totalHits.value(), trackTotalHitsUpTo), totalHitsRelation);
                 }
                 SearchResponse searchResponse = new SearchResponse(
                     SearchHits.empty(totalHits, Float.NaN),
@@ -1232,7 +1232,7 @@ public void testPartialAggsMixedWithFullResponses() {
                 SearchResponse mergedResponse = searchResponseMerger.getMergedResponse(clusters);
                 try {
                     SearchHits hits = mergedResponse.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(2L)); // should be 2 hits from remote1
+                    assertThat(hits.getTotalHits().value(), equalTo(2L)); // should be 2 hits from remote1
                     SearchHit hit1 = hits.getHits()[0];
                     String expectedHit1 = """
                         {
@@ -1273,7 +1273,7 @@ public void testPartialAggsMixedWithFullResponses() {
                 mergedResponse = searchResponseMerger.getMergedResponse(clusters);
                 try {
                     SearchHits hits = mergedResponse.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(4L)); // should be 2 hits from remote1, 2 from remote2
+                    assertThat(hits.getTotalHits().value(), equalTo(4L)); // should be 2 hits from remote1, 2 from remote2
 
                     SearchHit hit1 = hits.getHits()[0];
                     String expectedHit1 = """
@@ -1414,7 +1414,7 @@ public void testPartialAggsMixedWithFullResponses() {
                 mergedResponse = searchResponseMerger.getMergedResponse(clusters);
                 try {
                     SearchHits hits = mergedResponse.getHits();
-                    assertThat(hits.getTotalHits().value, equalTo(4L)); // should be 2 hits from remote1, 2 from remote2
+                    assertThat(hits.getTotalHits().value(), equalTo(4L)); // should be 2 hits from remote1, 2 from remote2
 
                     SearchHit hit1 = hits.getHits()[0];
                     String expectedHit1 = """
@@ -1483,7 +1483,7 @@ public void testPartialAggsMixedWithFullResponses() {
 
     private SearchHits createSimpleDeterministicSearchHits(String clusterAlias, Index[] indices) {
         TotalHits totalHits = new TotalHits(2, TotalHits.Relation.EQUAL_TO);
-        final int numDocs = (int) totalHits.value;
+        final int numDocs = (int) totalHits.value();
         int scoreFactor = 1;
         float maxScore = numDocs;
         int numFields = 1;
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
index afc4c6e9eccbf..bbeae6b19b8ac 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
@@ -621,8 +621,8 @@ public void testSerialization() throws IOException {
                 if (searchResponse.getHits().getTotalHits() == null) {
                     assertNull(deserialized.getHits().getTotalHits());
                 } else {
-                    assertEquals(searchResponse.getHits().getTotalHits().value, deserialized.getHits().getTotalHits().value);
-                    assertEquals(searchResponse.getHits().getTotalHits().relation, deserialized.getHits().getTotalHits().relation);
+                    assertEquals(searchResponse.getHits().getTotalHits().value(), deserialized.getHits().getTotalHits().value());
+                    assertEquals(searchResponse.getHits().getTotalHits().relation(), deserialized.getHits().getTotalHits().relation());
                 }
                 assertEquals(searchResponse.getHits().getHits().length, deserialized.getHits().getHits().length);
                 assertEquals(searchResponse.getNumReducePhases(), deserialized.getNumReducePhases());
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 06434a0c90518..70682cfd41d82 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -486,7 +486,7 @@ private MockTransportService[] startTransport(
                 threadPool
             );
             mockTransportServices[i] = remoteSeedTransport;
-            DiscoveryNode remoteSeedNode = remoteSeedTransport.getLocalDiscoNode();
+            DiscoveryNode remoteSeedNode = remoteSeedTransport.getLocalNode();
             knownNodes.add(remoteSeedNode);
             nodes[i] = remoteSeedNode;
             settingsBuilder.put("cluster.remote.remote" + i + ".seeds", remoteSeedNode.getAddress().toString());
diff --git a/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java b/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java
new file mode 100644
index 0000000000000..73d4ab59ce479
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class IndexComponentSelectorTests extends ESTestCase {
+
+    public void testIndexComponentSelectorFromKey() {
+        assertThat(IndexComponentSelector.getByKey("data"), equalTo(IndexComponentSelector.DATA));
+        assertThat(IndexComponentSelector.getByKey("failures"), equalTo(IndexComponentSelector.FAILURES));
+        assertThat(IndexComponentSelector.getByKey("*"), equalTo(IndexComponentSelector.ALL_APPLICABLE));
+        assertThat(IndexComponentSelector.getByKey("d*ta"), nullValue());
+        assertThat(IndexComponentSelector.getByKey("_all"), nullValue());
+        assertThat(IndexComponentSelector.getByKey("**"), nullValue());
+        assertThat(IndexComponentSelector.getByKey("failure"), nullValue());
+    }
+
+    public void testIndexComponentSelectorFromId() {
+        assertThat(IndexComponentSelector.getById((byte) 0), equalTo(IndexComponentSelector.DATA));
+        assertThat(IndexComponentSelector.getById((byte) 1), equalTo(IndexComponentSelector.FAILURES));
+        assertThat(IndexComponentSelector.getById((byte) 2), equalTo(IndexComponentSelector.ALL_APPLICABLE));
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> IndexComponentSelector.getById((byte) 3));
+        assertThat(
+            exception.getMessage(),
+            containsString("Unknown id of index component selector [3], available options are: {0=DATA, 1=FAILURES, 2=ALL_APPLICABLE}")
+        );
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index 437899a1fae1c..de7b43ad091fa 100644
--- a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.action.support;
 
 import org.elasticsearch.action.support.IndicesOptions.ConcreteTargetOptions;
-import org.elasticsearch.action.support.IndicesOptions.FailureStoreOptions;
 import org.elasticsearch.action.support.IndicesOptions.GatekeeperOptions;
 import org.elasticsearch.action.support.IndicesOptions.WildcardOptions;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -57,9 +56,7 @@ public void testSerialization() throws Exception {
                         .allowAliasToMultipleIndices(randomBoolean())
                         .allowClosedIndices(randomBoolean())
                 )
-                .failureStoreOptions(
-                    FailureStoreOptions.builder().includeRegularIndices(randomBoolean()).includeFailureIndices(randomBoolean())
-                )
+                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                 .build();
 
             BytesStreamOutput output = new BytesStreamOutput();
@@ -345,9 +342,9 @@ public void testToXContent() throws IOException {
             randomBoolean()
         );
         GatekeeperOptions gatekeeperOptions = new GatekeeperOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
-        FailureStoreOptions failureStoreOptions = new IndicesOptions.FailureStoreOptions(randomBoolean(), randomBoolean());
+        IndicesOptions.SelectorOptions selectorOptions = new IndicesOptions.SelectorOptions(randomFrom(IndexComponentSelector.values()));
 
-        IndicesOptions indicesOptions = new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, failureStoreOptions);
+        IndicesOptions indicesOptions = new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions);
 
         XContentType type = randomFrom(XContentType.values());
         BytesReference xContentBytes = toXContentBytes(indicesOptions, type);
@@ -362,7 +359,15 @@ public void testToXContent() throws IOException {
         assertThat(map.get("ignore_unavailable"), equalTo(concreteTargetOptions.allowUnavailableTargets()));
         assertThat(map.get("allow_no_indices"), equalTo(wildcardOptions.allowEmptyExpressions()));
         assertThat(map.get("ignore_throttled"), equalTo(gatekeeperOptions.ignoreThrottled()));
-        assertThat(map.get("failure_store"), equalTo(failureStoreOptions.displayValue()));
+        String displayValue;
+        if (IndicesOptions.SelectorOptions.ALL_APPLICABLE.equals(selectorOptions)) {
+            displayValue = "include";
+        } else if (IndicesOptions.SelectorOptions.DATA.equals(selectorOptions)) {
+            displayValue = "exclude";
+        } else {
+            displayValue = "only";
+        }
+        assertThat(map.get("failure_store"), equalTo(displayValue));
     }
 
     public void testFromXContent() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java b/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
index c00fece686524..1b86e5b00000c 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
@@ -400,6 +400,6 @@ protected Fields getTermVectorsFromLucene(DirectoryReader directoryReader, TestD
 
         ScoreDoc[] scoreDocs = search.scoreDocs;
         assertEquals(1, scoreDocs.length);
-        return directoryReader.getTermVectors(scoreDocs[0].doc);
+        return directoryReader.termVectors().get(scoreDocs[0].doc);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java
index d1734dbf45f04..ca7376a43d718 100644
--- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java
@@ -424,7 +424,7 @@ public void testExpandsTimestampRangeForPrimary() throws Exception {
         }
 
         final var eventIngestedRange = resultingState.metadata().index(indexName).getEventIngestedRange();
-        if (clusterState.getMinTransportVersion().before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+        if (clusterState.getMinTransportVersion().before(TransportVersions.V_8_15_0)) {
             assertThat(eventIngestedRange, sameInstance(IndexLongFieldRange.UNKNOWN));
         } else {
             if (shardEventIngestedRange == ShardLongFieldRange.UNKNOWN) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index a1536e17a15de..3c680d891ff13 100644
--- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -599,7 +599,7 @@ public void testStartedShardEntrySerialization() throws Exception {
             assertThat(deserialized.primaryTerm, equalTo(primaryTerm));
             assertThat(deserialized.message, equalTo(message));
             assertThat(deserialized.timestampRange, equalTo(timestampRange));
-            if (version.before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) {
+            if (version.before(TransportVersions.V_8_15_0)) {
                 assertThat(deserialized.eventIngestedRange, equalTo(ShardLongFieldRange.UNKNOWN));
             } else {
                 assertThat(deserialized.eventIngestedRange, equalTo(eventIngestedRange));
@@ -615,7 +615,7 @@ public void testStartedShardEntrySerializationWithOlderTransportVersion() throws
         final TransportVersion version = randomFrom(
             getFirstVersion(),
             getPreviousVersion(TransportVersions.MINIMUM_COMPATIBLE),
-            getPreviousVersion(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)
+            getPreviousVersion(TransportVersions.V_8_15_0)
         );
         final ShardLongFieldRange timestampRange = ShardLongFieldRangeWireTests.randomRange();
         final ShardLongFieldRange eventIngestedRange = ShardLongFieldRangeWireTests.randomRange();
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinitionTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinitionTests.java
new file mode 100644
index 0000000000000..38d4031755a55
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinitionTests.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.routing.allocation.DataTier;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.cluster.metadata.DataStreamFailureStoreDefinition.INDEX_FAILURE_STORE_VERSION_SETTING_NAME;
+import static org.hamcrest.Matchers.equalTo;
+
+public class DataStreamFailureStoreDefinitionTests extends ESTestCase {
+
+    public void testSettingsFiltering() {
+        // Empty
+        Settings.Builder builder = Settings.builder();
+        Settings.Builder expectedBuilder = Settings.builder();
+        assertThat(DataStreamFailureStoreDefinition.filterUserDefinedSettings(builder).keys(), equalTo(expectedBuilder.keys()));
+
+        // All supported settings
+        builder.put(INDEX_FAILURE_STORE_VERSION_SETTING_NAME, 3)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(DataTier.TIER_PREFERENCE, "data_cold")
+            .put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
+            .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-10")
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s")
+            .put(IndexMetadata.LIFECYCLE_NAME, "my-policy")
+            .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "." + randomAlphaOfLength(4), randomAlphaOfLength(4))
+            .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "." + randomAlphaOfLength(4), randomAlphaOfLength(4))
+            .put(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "." + randomAlphaOfLength(4), randomAlphaOfLength(4));
+        // We expect no changes
+        expectedBuilder = Settings.builder().put(builder.build());
+        assertThat(DataStreamFailureStoreDefinition.filterUserDefinedSettings(builder).keys(), equalTo(expectedBuilder.keys()));
+
+        // Remove unsupported settings
+        String randomSetting = randomAlphaOfLength(10);
+        builder.put(INDEX_FAILURE_STORE_VERSION_SETTING_NAME, 3)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(DataTier.TIER_PREFERENCE, "data_cold")
+            .put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
+            .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-10")
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s")
+            .put(IndexMetadata.LIFECYCLE_NAME, "my-policy")
+            .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "." + randomAlphaOfLength(4), randomAlphaOfLength(4))
+            .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "." + randomAlphaOfLength(4), randomAlphaOfLength(4))
+            .put(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "." + randomAlphaOfLength(4), randomAlphaOfLength(4))
+            .put(IndexSettings.MODE.getKey(), randomFrom(IndexMode.values()))
+            .put(randomSetting, randomAlphaOfLength(10));
+        // We expect no changes
+        expectedBuilder = Settings.builder().put(builder.build());
+        assertThat(
+            DataStreamFailureStoreDefinition.filterUserDefinedSettings(builder).keys().size(),
+            equalTo(expectedBuilder.keys().size() - 2)
+        );
+        assertThat(
+            DataStreamFailureStoreDefinition.filterUserDefinedSettings(builder).keys().contains(IndexSettings.MODE.getKey()),
+            equalTo(false)
+        );
+        assertThat(DataStreamFailureStoreDefinition.filterUserDefinedSettings(builder).keys().contains(randomSetting), equalTo(false));
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java
index 020955d226a0f..9b0eb93b496a4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java
@@ -24,7 +24,16 @@ protected Writeable.Reader<DataStreamOptions> instanceReader() {
 
     @Override
     protected DataStreamOptions createTestInstance() {
-        return new DataStreamOptions(randomBoolean() ? null : DataStreamFailureStoreTests.randomFailureStore());
+        return randomDataStreamOptions();
+    }
+
+    public static DataStreamOptions randomDataStreamOptions() {
+        return switch (randomIntBetween(0, 2)) {
+            case 0 -> DataStreamOptions.EMPTY;
+            case 1 -> DataStreamOptions.FAILURE_STORE_DISABLED;
+            case 2 -> DataStreamOptions.FAILURE_STORE_ENABLED;
+            default -> throw new IllegalArgumentException("Illegal randomisation branch");
+        };
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java
deleted file mode 100644
index 1ca59ff402bd8..0000000000000
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ExpressionList;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ExpressionList.Expression;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.function.Supplier;
-
-import static org.hamcrest.Matchers.is;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class ExpressionListTests extends ESTestCase {
-
-    public void testEmpty() {
-        ExpressionList expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), List.of());
-        assertThat(expressionList.iterator().hasNext(), is(false));
-        assertThat(expressionList.hasWildcard(), is(false));
-        expressionList = new ExpressionList(getContextWithOptions(getNoExpandWildcardsIndicesOptions()), List.of());
-        assertThat(expressionList.iterator().hasNext(), is(false));
-        assertThat(expressionList.hasWildcard(), is(false));
-    }
-
-    public void testExplicitSingleNameExpression() {
-        for (IndicesOptions indicesOptions : List.of(getExpandWildcardsIndicesOptions(), getNoExpandWildcardsIndicesOptions())) {
-            for (String expressionString : List.of("non_wildcard", "-non_exclusion")) {
-                ExpressionList expressionList = new ExpressionList(getContextWithOptions(indicesOptions), List.of(expressionString));
-                assertThat(expressionList.hasWildcard(), is(false));
-                if (randomBoolean()) {
-                    expressionList = new ExpressionList(getContextWithOptions(indicesOptions), List.of(expressionString));
-                }
-                Iterator<Expression> expressionIterator = expressionList.iterator();
-                assertThat(expressionIterator.hasNext(), is(true));
-                if (randomBoolean()) {
-                    expressionIterator = expressionList.iterator();
-                }
-                Expression expression = expressionIterator.next();
-                assertThat(expression.isExclusion(), is(false));
-                assertThat(expression.isWildcard(), is(false));
-                assertThat(expression.get(), is(expressionString));
-                assertThat(expressionIterator.hasNext(), is(false));
-            }
-        }
-    }
-
-    public void testWildcardSingleExpression() {
-        for (String wildcardTest : List.of("*", "a*", "*b", "a*b", "a-*b", "a*-b", "-*", "-a*", "-*b", "**", "*-*")) {
-            ExpressionList expressionList = new ExpressionList(
-                getContextWithOptions(getExpandWildcardsIndicesOptions()),
-                List.of(wildcardTest)
-            );
-            assertThat(expressionList.hasWildcard(), is(true));
-            if (randomBoolean()) {
-                expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), List.of(wildcardTest));
-            }
-            Iterator<Expression> expressionIterator = expressionList.iterator();
-            assertThat(expressionIterator.hasNext(), is(true));
-            if (randomBoolean()) {
-                expressionIterator = expressionList.iterator();
-            }
-            Expression expression = expressionIterator.next();
-            assertThat(expression.isExclusion(), is(false));
-            assertThat(expression.isWildcard(), is(true));
-            assertThat(expression.get(), is(wildcardTest));
-            assertThat(expressionIterator.hasNext(), is(false));
-        }
-    }
-
-    public void testWildcardLongerExpression() {
-        List<String> onlyExplicits = randomList(7, () -> randomAlphaOfLengthBetween(0, 5));
-        String wildcard = randomFrom("*", "*b", "-*", "*-", "c*", "a*b", "**");
-        List<String> expressionList = new ArrayList<>(onlyExplicits.size() + 1);
-        expressionList.addAll(randomSubsetOf(onlyExplicits));
-        int wildcardPos = expressionList.size();
-        expressionList.add(wildcard);
-        for (String item : onlyExplicits) {
-            if (expressionList.contains(item) == false) {
-                expressionList.add(item);
-            }
-        }
-        ExpressionList expressionIterable = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), expressionList);
-        assertThat(expressionIterable.hasWildcard(), is(true));
-        if (randomBoolean()) {
-            expressionIterable = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), expressionList);
-        }
-        int i = 0;
-        for (Expression expression : expressionIterable) {
-            assertThat(expression.isExclusion(), is(false));
-            if (i != wildcardPos) {
-                assertThat(expression.isWildcard(), is(false));
-            } else {
-                assertThat(expression.isWildcard(), is(true));
-            }
-            assertThat(expression.get(), is(expressionList.get(i++)));
-        }
-    }
-
-    public void testWildcardsNoExclusionExpressions() {
-        for (List<String> wildcardExpression : List.of(
-            List.of("*"),
-            List.of("a", "*"),
-            List.of("-b", "*c"),
-            List.of("-", "a", "c*"),
-            List.of("*", "a*", "*b"),
-            List.of("-*", "a", "b*")
-        )) {
-            ExpressionList expressionList = new ExpressionList(
-                getContextWithOptions(getExpandWildcardsIndicesOptions()),
-                wildcardExpression
-            );
-            assertThat(expressionList.hasWildcard(), is(true));
-            if (randomBoolean()) {
-                expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), wildcardExpression);
-            }
-            int i = 0;
-            for (Expression expression : expressionList) {
-                assertThat(expression.isExclusion(), is(false));
-                if (wildcardExpression.get(i).contains("*")) {
-                    assertThat(expression.isWildcard(), is(true));
-                } else {
-                    assertThat(expression.isWildcard(), is(false));
-                }
-                assertThat(expression.get(), is(wildcardExpression.get(i++)));
-            }
-        }
-    }
-
-    public void testWildcardExpressionNoExpandOptions() {
-        for (List<String> wildcardExpression : List.of(
-            List.of("*"),
-            List.of("a", "*"),
-            List.of("-b", "*c"),
-            List.of("*d", "-"),
-            List.of("*", "-*"),
-            List.of("-", "a", "c*"),
-            List.of("*", "a*", "*b")
-        )) {
-            ExpressionList expressionList = new ExpressionList(
-                getContextWithOptions(getNoExpandWildcardsIndicesOptions()),
-                wildcardExpression
-            );
-            assertThat(expressionList.hasWildcard(), is(false));
-            if (randomBoolean()) {
-                expressionList = new ExpressionList(getContextWithOptions(getNoExpandWildcardsIndicesOptions()), wildcardExpression);
-            }
-            int i = 0;
-            for (Expression expression : expressionList) {
-                assertThat(expression.isWildcard(), is(false));
-                assertThat(expression.isExclusion(), is(false));
-                assertThat(expression.get(), is(wildcardExpression.get(i++)));
-            }
-        }
-    }
-
-    public void testSingleExclusionExpression() {
-        String wildcard = randomFrom("*", "*b", "-*", "*-", "c*", "a*b", "**", "*-*");
-        int wildcardPos = randomIntBetween(0, 3);
-        String exclusion = randomFrom("-*", "-", "-c*", "-ab", "--");
-        int exclusionPos = randomIntBetween(wildcardPos + 1, 7);
-        List<String> exclusionExpression = new ArrayList<>();
-        for (int i = 0; i < wildcardPos; i++) {
-            exclusionExpression.add(randomAlphaOfLengthBetween(0, 5));
-        }
-        exclusionExpression.add(wildcard);
-        for (int i = wildcardPos + 1; i < exclusionPos; i++) {
-            exclusionExpression.add(randomAlphaOfLengthBetween(0, 5));
-        }
-        exclusionExpression.add(exclusion);
-        for (int i = 0; i < randomIntBetween(0, 3); i++) {
-            exclusionExpression.add(randomAlphaOfLengthBetween(0, 5));
-        }
-        ExpressionList expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), exclusionExpression);
-        if (randomBoolean()) {
-            assertThat(expressionList.hasWildcard(), is(true));
-        }
-        int i = 0;
-        for (Expression expression : expressionList) {
-            if (i == wildcardPos) {
-                assertThat(expression.isWildcard(), is(true));
-                assertThat(expression.isExclusion(), is(false));
-                assertThat(expression.get(), is(exclusionExpression.get(i++)));
-            } else if (i == exclusionPos) {
-                assertThat(expression.isExclusion(), is(true));
-                assertThat(expression.isWildcard(), is(exclusionExpression.get(i).contains("*")));
-                assertThat(expression.get(), is(exclusionExpression.get(i++).substring(1)));
-            } else {
-                assertThat(expression.isWildcard(), is(false));
-                assertThat(expression.isExclusion(), is(false));
-                assertThat(expression.get(), is(exclusionExpression.get(i++)));
-            }
-        }
-    }
-
-    public void testExclusionsExpression() {
-        for (Tuple<List<String>, List<Boolean>> exclusionExpression : List.of(
-            new Tuple<>(List.of("-a", "*", "-a"), List.of(false, false, true)),
-            new Tuple<>(List.of("-b*", "c", "-a"), List.of(false, false, true)),
-            new Tuple<>(List.of("*d", "-", "*b"), List.of(false, true, false)),
-            new Tuple<>(List.of("-", "--", "-*", "", "-*"), List.of(false, false, false, false, true)),
-            new Tuple<>(List.of("*-", "-*", "a", "-b"), List.of(false, true, false, true)),
-            new Tuple<>(List.of("a", "-b", "-*", "-b", "*", "-b"), List.of(false, false, false, true, false, true)),
-            new Tuple<>(List.of("-a", "*d", "-a", "-*b", "-b", "--"), List.of(false, false, true, true, true, true))
-        )) {
-            ExpressionList expressionList = new ExpressionList(
-                getContextWithOptions(getExpandWildcardsIndicesOptions()),
-                exclusionExpression.v1()
-            );
-            if (randomBoolean()) {
-                assertThat(expressionList.hasWildcard(), is(true));
-            }
-            int i = 0;
-            for (Expression expression : expressionList) {
-                boolean isExclusion = exclusionExpression.v2().get(i);
-                assertThat(expression.isExclusion(), is(isExclusion));
-                assertThat(expression.isWildcard(), is(exclusionExpression.v1().get(i).contains("*")));
-                if (isExclusion) {
-                    assertThat(expression.get(), is(exclusionExpression.v1().get(i++).substring(1)));
-                } else {
-                    assertThat(expression.get(), is(exclusionExpression.v1().get(i++)));
-                }
-            }
-        }
-    }
-
-    private IndicesOptions getExpandWildcardsToOpenOnlyIndicesOptions() {
-        return IndicesOptions.fromOptions(
-            randomBoolean(),
-            randomBoolean(),
-            true,
-            false,
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean()
-        );
-    }
-
-    private IndicesOptions getExpandWildcardsToCloseOnlyIndicesOptions() {
-        return IndicesOptions.fromOptions(
-            randomBoolean(),
-            randomBoolean(),
-            false,
-            true,
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean()
-        );
-    }
-
-    private IndicesOptions getExpandWildcardsToOpenCloseIndicesOptions() {
-        return IndicesOptions.fromOptions(
-            randomBoolean(),
-            randomBoolean(),
-            true,
-            true,
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean()
-        );
-    }
-
-    private IndicesOptions getExpandWildcardsIndicesOptions() {
-        return ESTestCase.<Supplier<IndicesOptions>>randomFrom(
-            this::getExpandWildcardsToOpenOnlyIndicesOptions,
-            this::getExpandWildcardsToCloseOnlyIndicesOptions,
-            this::getExpandWildcardsToOpenCloseIndicesOptions
-        ).get();
-    }
-
-    private IndicesOptions getNoExpandWildcardsIndicesOptions() {
-        return IndicesOptions.fromOptions(
-            randomBoolean(),
-            randomBoolean(),
-            false,
-            false,
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean(),
-            randomBoolean()
-        );
-    }
-
-    private Context getContextWithOptions(IndicesOptions indicesOptions) {
-        Context context = mock(Context.class);
-        when(context.getOptions()).thenReturn(indicesOptions);
-        return context;
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 5f55d203e00e4..99470918ce063 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -2298,7 +2298,8 @@ public void testIgnoreThrottled() {
                 new IndicesOptions(
                     IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
                     IndicesOptions.WildcardOptions.DEFAULT,
-                    IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build()
+                    IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build(),
+                    IndicesOptions.SelectorOptions.DEFAULT
                 ),
                 "ind*",
                 "test-index"
@@ -2736,7 +2737,7 @@ public void testDataStreamsWithFailureStore() {
         // Test include failure store with an exact data stream name
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                 .build();
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream");
             assertThat(result.length, equalTo(4));
@@ -2750,7 +2751,7 @@ public void testDataStreamsWithFailureStore() {
         // We expect that they will be skipped
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                 .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowFailureIndices(false).build())
                 .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
                 .build();
@@ -2764,7 +2765,7 @@ public void testDataStreamsWithFailureStore() {
         // We expect an error
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                 .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowFailureIndices(false).build())
                 .build();
             FailureIndexNotSupportedException failureIndexNotSupportedException = expectThrows(
@@ -2780,7 +2781,7 @@ public void testDataStreamsWithFailureStore() {
         // Test only failure store with an exact data stream name
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(false).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
                 .build();
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream");
             assertThat(result.length, equalTo(2));
@@ -2807,7 +2808,7 @@ public void testDataStreamsWithFailureStore() {
         // Test include failure store without any expressions
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                 .build();
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true);
             assertThat(result.length, equalTo(5));
@@ -2827,7 +2828,7 @@ public void testDataStreamsWithFailureStore() {
         // Test only failure store without any expressions
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(false).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
                 .build();
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true);
             assertThat(result.length, equalTo(2));
@@ -2860,7 +2861,7 @@ public void testDataStreamsWithFailureStore() {
         // Test include failure store with wildcard expression
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                 .build();
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*");
             assertThat(result.length, equalTo(5));
@@ -2880,7 +2881,7 @@ public void testDataStreamsWithFailureStore() {
         // Test only failure store with wildcard expression
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(false).includeFailureIndices(true))
+                .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
                 .build();
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*");
             assertThat(result.length, equalTo(2));
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index b8804e9160a75..96a74d2e23aad 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -9,6 +9,8 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.TransportVersion;
@@ -42,8 +44,10 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
@@ -72,6 +76,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -602,6 +607,8 @@ public void testCalculateNumRoutingShards() {
     public void testValidateDotIndex() {
         List<SystemIndexDescriptor> systemIndexDescriptors = new ArrayList<>();
         systemIndexDescriptors.add(SystemIndexDescriptorUtils.createUnmanaged(".test-one*", "test"));
+        Automaton patternAutomaton = new RegExp("\\.test-~(one.*)", RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton();
+
         systemIndexDescriptors.add(SystemIndexDescriptorUtils.createUnmanaged(".test-~(one*)", "test"));
         systemIndexDescriptors.add(SystemIndexDescriptorUtils.createUnmanaged(".pattern-test*", "test-1"));
 
@@ -687,6 +694,178 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() {
         assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("value2"));
     }
 
+    public void testAggregateSettingsProviderOverrulesSettingsFromRequest() {
+        IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> {
+            builder.settings(Settings.builder().put("template_setting", "value1"));
+        });
+        Metadata metadata = new Metadata.Builder().templates(Map.of("template_1", templateMetadata)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
+        request.settings(Settings.builder().put("request_setting", "value2").build());
+
+        Settings aggregatedIndexSettings = aggregateIndexSettings(
+            clusterState,
+            request,
+            templateMetadata.settings(),
+            null,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Set.of(new IndexSettingProvider() {
+                @Override
+                public Settings getAdditionalIndexSettings(
+                    String indexName,
+                    String dataStreamName,
+                    IndexMode templateIndexMode,
+                    Metadata metadata,
+                    Instant resolvedAt,
+                    Settings indexTemplateAndCreateRequestSettings,
+                    List<CompressedXContent> combinedTemplateMappings
+                ) {
+                    return Settings.builder().put("request_setting", "overrule_value").put("other_setting", "other_value").build();
+                }
+
+                @Override
+                public boolean overrulesTemplateAndRequestSettings() {
+                    return true;
+                }
+            })
+        );
+
+        assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1"));
+        assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("overrule_value"));
+        assertThat(aggregatedIndexSettings.get("other_setting"), equalTo("other_value"));
+    }
+
+    public void testAggregateSettingsProviderOverrulesNullFromRequest() {
+        IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> {
+            builder.settings(Settings.builder().put("template_setting", "value1"));
+        });
+        Metadata metadata = new Metadata.Builder().templates(Map.of("template_1", templateMetadata)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
+        request.settings(Settings.builder().putNull("request_setting").build());
+
+        Settings aggregatedIndexSettings = aggregateIndexSettings(
+            clusterState,
+            request,
+            templateMetadata.settings(),
+            null,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Set.of(new IndexSettingProvider() {
+                @Override
+                public Settings getAdditionalIndexSettings(
+                    String indexName,
+                    String dataStreamName,
+                    IndexMode templateIndexMode,
+                    Metadata metadata,
+                    Instant resolvedAt,
+                    Settings indexTemplateAndCreateRequestSettings,
+                    List<CompressedXContent> combinedTemplateMappings
+                ) {
+                    return Settings.builder().put("request_setting", "overrule_value").put("other_setting", "other_value").build();
+                }
+
+                @Override
+                public boolean overrulesTemplateAndRequestSettings() {
+                    return true;
+                }
+            })
+        );
+
+        assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1"));
+        assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("overrule_value"));
+        assertThat(aggregatedIndexSettings.get("other_setting"), equalTo("other_value"));
+    }
+
+    public void testAggregateSettingsProviderOverrulesSettingsFromTemplates() {
+        IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> {
+            builder.settings(Settings.builder().put("template_setting", "value1"));
+        });
+        Metadata metadata = new Metadata.Builder().templates(Map.of("template_1", templateMetadata)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
+        request.settings(Settings.builder().put("request_setting", "value2").build());
+
+        Settings aggregatedIndexSettings = aggregateIndexSettings(
+            clusterState,
+            request,
+            templateMetadata.settings(),
+            null,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Set.of(new IndexSettingProvider() {
+                @Override
+                public Settings getAdditionalIndexSettings(
+                    String indexName,
+                    String dataStreamName,
+                    IndexMode templateIndexMode,
+                    Metadata metadata,
+                    Instant resolvedAt,
+                    Settings indexTemplateAndCreateRequestSettings,
+                    List<CompressedXContent> combinedTemplateMappings
+                ) {
+                    return Settings.builder().put("template_setting", "overrule_value").put("other_setting", "other_value").build();
+                }
+
+                @Override
+                public boolean overrulesTemplateAndRequestSettings() {
+                    return true;
+                }
+            })
+        );
+
+        assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("overrule_value"));
+        assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("value2"));
+        assertThat(aggregatedIndexSettings.get("other_setting"), equalTo("other_value"));
+    }
+
+    public void testAggregateSettingsProviderOverrulesNullFromTemplates() {
+        IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> {
+            builder.settings(Settings.builder().putNull("template_setting"));
+        });
+        Metadata metadata = new Metadata.Builder().templates(Map.of("template_1", templateMetadata)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
+        request.settings(Settings.builder().put("request_setting", "value2").build());
+
+        Settings aggregatedIndexSettings = aggregateIndexSettings(
+            clusterState,
+            request,
+            templateMetadata.settings(),
+            null,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Set.of(new IndexSettingProvider() {
+                @Override
+                public Settings getAdditionalIndexSettings(
+                    String indexName,
+                    String dataStreamName,
+                    IndexMode templateIndexMode,
+                    Metadata metadata,
+                    Instant resolvedAt,
+                    Settings indexTemplateAndCreateRequestSettings,
+                    List<CompressedXContent> combinedTemplateMappings
+                ) {
+                    return Settings.builder().put("template_setting", "overrule_value").put("other_setting", "other_value").build();
+                }
+
+                @Override
+                public boolean overrulesTemplateAndRequestSettings() {
+                    return true;
+                }
+            })
+        );
+
+        assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("overrule_value"));
+        assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("value2"));
+        assertThat(aggregatedIndexSettings.get("other_setting"), equalTo("other_value"));
+    }
+
     public void testInvalidAliasName() {
         final String[] invalidAliasNames = new String[] { "-alias1", "+alias2", "_alias3", "a#lias", "al:ias", ".", ".." };
         String aliasName = randomFrom(invalidAliasNames);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
index 92c1103c950c0..276c20d2d1322 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
@@ -422,6 +422,39 @@ public void testUpdateLifecycle() {
         }
     }
 
+    public void testUpdateDataStreamOptions() {
+        String dataStream = randomAlphaOfLength(5);
+        // we want the data stream options to be non-empty, so we can see the removal in action
+        DataStreamOptions dataStreamOptions = randomValueOtherThan(
+            DataStreamOptions.EMPTY,
+            DataStreamOptionsTests::randomDataStreamOptions
+        );
+        ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStream, 2)), List.of());
+        MetadataDataStreamsService service = new MetadataDataStreamsService(
+            mock(ClusterService.class),
+            mock(IndicesService.class),
+            DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings())
+        );
+
+        // Ensure no data stream options are stored
+        DataStream updatedDataStream = before.metadata().dataStreams().get(dataStream);
+        assertNotNull(updatedDataStream);
+        assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
+
+        // Set non-empty data stream options
+        ClusterState after = service.updateDataStreamOptions(before, List.of(dataStream), dataStreamOptions);
+        updatedDataStream = after.metadata().dataStreams().get(dataStream);
+        assertNotNull(updatedDataStream);
+        assertThat(updatedDataStream.getDataStreamOptions(), equalTo(dataStreamOptions));
+        before = after;
+
+        // Remove data stream options
+        after = service.updateDataStreamOptions(before, List.of(dataStream), null);
+        updatedDataStream = after.metadata().dataStreams().get(dataStream);
+        assertNotNull(updatedDataStream);
+        assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
+    }
+
     private MapperService getMapperService(IndexMetadata im) {
         try {
             String mapping = im.mapping().source().toString();
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
index 00e21603ec8b4..ba1f9f01f49d2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
@@ -2412,30 +2413,87 @@ public void testEnsureMetadataFieldCheckedForGlobalStateChanges() {
         assertThat(unclassifiedFields, empty());
     }
 
-    public void testIsTimeSeriesTemplate() throws IOException {
-        var template = new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("{}"), null);
+    public void testRetrieveIndexModeFromTemplateTsdb() throws IOException {
+        // tsdb:
+        var tsdbTemplate = new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("{}"), null);
         // Settings in component template:
         {
-            var componentTemplate = new ComponentTemplate(template, null, null);
+            var componentTemplate = new ComponentTemplate(tsdbTemplate, null, null);
             var indexTemplate = ComposableIndexTemplate.builder()
                 .indexPatterns(List.of("test-*"))
                 .componentTemplates(List.of("component_template_1"))
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
                 .build();
             Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build();
-            assertThat(m.isTimeSeriesTemplate(indexTemplate), is(true));
+            assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES));
         }
         // Settings in composable index template:
         {
             var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null);
             var indexTemplate = ComposableIndexTemplate.builder()
                 .indexPatterns(List.of("test-*"))
-                .template(template)
+                .template(tsdbTemplate)
                 .componentTemplates(List.of("component_template_1"))
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
                 .build();
             Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build();
-            assertThat(m.isTimeSeriesTemplate(indexTemplate), is(true));
+            assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES));
+        }
+    }
+
+    public void testRetrieveIndexModeFromTemplateLogsdb() throws IOException {
+        // logsdb:
+        var logsdbTemplate = new Template(Settings.builder().put("index.mode", "logsdb").build(), new CompressedXContent("{}"), null);
+        // Settings in component template:
+        {
+            var componentTemplate = new ComponentTemplate(logsdbTemplate, null, null);
+            var indexTemplate = ComposableIndexTemplate.builder()
+                .indexPatterns(List.of("test-*"))
+                .componentTemplates(List.of("component_template_1"))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .build();
+            Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build();
+            assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB));
+        }
+        // Settings in composable index template:
+        {
+            var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null);
+            var indexTemplate = ComposableIndexTemplate.builder()
+                .indexPatterns(List.of("test-*"))
+                .template(logsdbTemplate)
+                .componentTemplates(List.of("component_template_1"))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .build();
+            Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build();
+            assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB));
+        }
+    }
+
+    public void testRetrieveIndexModeFromTemplateEmpty() throws IOException {
+        // no index mode:
+        var emptyTemplate = new Template(Settings.EMPTY, new CompressedXContent("{}"), null);
+        // Settings in component template:
+        {
+            var componentTemplate = new ComponentTemplate(emptyTemplate, null, null);
+            var indexTemplate = ComposableIndexTemplate.builder()
+                .indexPatterns(List.of("test-*"))
+                .componentTemplates(List.of("component_template_1"))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .build();
+            Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build();
+            assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), nullValue());
+        }
+        // Settings in composable index template:
+        {
+            var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null);
+            var indexTemplate = ComposableIndexTemplate.builder()
+                .indexPatterns(List.of("test-*"))
+                .template(emptyTemplate)
+                .componentTemplates(List.of("component_template_1"))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .build();
+            Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build();
+            assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), nullValue());
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java
index 6a7f4bb27a324..21b30557cafea 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.cluster.routing;
 
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
@@ -20,7 +19,6 @@
 
 import java.util.List;
 
-import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO;
 import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
@@ -29,22 +27,16 @@
 public class IndexRoutingTableTests extends ESTestCase {
 
     public void testReadyForSearch() {
-        innerReadyForSearch(false, false);
-        innerReadyForSearch(false, true);
-        innerReadyForSearch(true, false);
-        innerReadyForSearch(true, true);
+        innerReadyForSearch(false);
+        innerReadyForSearch(true);
     }
 
-    // TODO: remove if (fastRefresh && beforeFastRefreshRCO) branches (ES-9563)
-    private void innerReadyForSearch(boolean fastRefresh, boolean beforeFastRefreshRCO) {
+    private void innerReadyForSearch(boolean fastRefresh) {
         Index index = new Index(randomIdentifier(), UUIDs.randomBase64UUID());
         ClusterState clusterState = mock(ClusterState.class, Mockito.RETURNS_DEEP_STUBS);
         when(clusterState.metadata().index(any(Index.class)).getSettings()).thenReturn(
             Settings.builder().put(INDEX_FAST_REFRESH_SETTING.getKey(), fastRefresh).build()
         );
-        when(clusterState.getMinTransportVersion()).thenReturn(
-            beforeFastRefreshRCO ? TransportVersion.fromId(FAST_REFRESH_RCO.id() - 1_00_0) : TransportVersion.current()
-        );
         // 2 primaries that are search and index
         ShardId p1 = new ShardId(index, 0);
         IndexShardRoutingTable shardTable1 = new IndexShardRoutingTable(
@@ -63,7 +55,7 @@ private void innerReadyForSearch(boolean fastRefresh, boolean beforeFastRefreshR
         shardTable1 = new IndexShardRoutingTable(p1, List.of(getShard(p1, true, ShardRoutingState.STARTED, ShardRouting.Role.INDEX_ONLY)));
         shardTable2 = new IndexShardRoutingTable(p2, List.of(getShard(p2, true, ShardRoutingState.STARTED, ShardRouting.Role.INDEX_ONLY)));
         indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 });
-        if (fastRefresh && beforeFastRefreshRCO) {
+        if (fastRefresh) {
             assertTrue(indexRoutingTable.readyForSearch(clusterState));
         } else {
             assertFalse(indexRoutingTable.readyForSearch(clusterState));
@@ -99,7 +91,7 @@ private void innerReadyForSearch(boolean fastRefresh, boolean beforeFastRefreshR
             )
         );
         indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 });
-        if (fastRefresh && beforeFastRefreshRCO) {
+        if (fastRefresh) {
             assertTrue(indexRoutingTable.readyForSearch(clusterState));
         } else {
             assertFalse(indexRoutingTable.readyForSearch(clusterState));
@@ -126,6 +118,8 @@ private void innerReadyForSearch(boolean fastRefresh, boolean beforeFastRefreshR
         assertTrue(indexRoutingTable.readyForSearch(clusterState));
 
         // 2 unassigned primaries that are index only with some replicas that are all available
+        // Fast refresh indices do not support replicas so this can not practically happen. If we add support we will want to ensure
+        // that readyForSearch allows for searching replicas when the index shard is not available.
         shardTable1 = new IndexShardRoutingTable(
             p1,
             List.of(
@@ -143,8 +137,8 @@ private void innerReadyForSearch(boolean fastRefresh, boolean beforeFastRefreshR
             )
         );
         indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 });
-        if (fastRefresh && beforeFastRefreshRCO) {
-            assertFalse(indexRoutingTable.readyForSearch(clusterState));
+        if (fastRefresh) {
+            assertFalse(indexRoutingTable.readyForSearch(clusterState)); // if we support replicas for fast refreshes this needs to change
         } else {
             assertTrue(indexRoutingTable.readyForSearch(clusterState));
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetricsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetricsTests.java
new file mode 100644
index 0000000000000..2c642da665051
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetricsTests.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.routing.allocation.allocator;
+
+import org.elasticsearch.telemetry.InstrumentType;
+import org.elasticsearch.telemetry.RecordingMeterRegistry;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+
+public class DesiredBalanceMetricsTests extends ESTestCase {
+
+    public void testZeroAllMetrics() {
+        DesiredBalanceMetrics metrics = new DesiredBalanceMetrics(MeterRegistry.NOOP);
+        long unassignedShards = randomNonNegativeLong();
+        long totalAllocations = randomNonNegativeLong();
+        long undesiredAllocations = randomNonNegativeLong();
+        metrics.updateMetrics(unassignedShards, totalAllocations, undesiredAllocations);
+        assertEquals(totalAllocations, metrics.totalAllocations());
+        assertEquals(unassignedShards, metrics.unassignedShards());
+        assertEquals(undesiredAllocations, metrics.undesiredAllocations());
+        metrics.zeroAllMetrics();
+        assertEquals(0, metrics.totalAllocations());
+        assertEquals(0, metrics.unassignedShards());
+        assertEquals(0, metrics.undesiredAllocations());
+    }
+
+    public void testMetricsAreOnlyPublishedWhenNodeIsMaster() {
+        RecordingMeterRegistry meterRegistry = new RecordingMeterRegistry();
+        DesiredBalanceMetrics metrics = new DesiredBalanceMetrics(meterRegistry);
+
+        long unassignedShards = randomNonNegativeLong();
+        long totalAllocations = randomLongBetween(100, 10000000);
+        long undesiredAllocations = randomLongBetween(0, totalAllocations);
+        metrics.updateMetrics(unassignedShards, totalAllocations, undesiredAllocations);
+
+        // Collect when not master
+        meterRegistry.getRecorder().collect();
+        assertThat(
+            meterRegistry.getRecorder()
+                .getMeasurements(InstrumentType.LONG_GAUGE, DesiredBalanceMetrics.UNDESIRED_ALLOCATION_COUNT_METRIC_NAME),
+            empty()
+        );
+        assertThat(
+            meterRegistry.getRecorder().getMeasurements(InstrumentType.LONG_GAUGE, DesiredBalanceMetrics.TOTAL_SHARDS_METRIC_NAME),
+            empty()
+        );
+        assertThat(
+            meterRegistry.getRecorder().getMeasurements(InstrumentType.LONG_GAUGE, DesiredBalanceMetrics.UNASSIGNED_SHARDS_METRIC_NAME),
+            empty()
+        );
+        assertThat(
+            meterRegistry.getRecorder()
+                .getMeasurements(InstrumentType.DOUBLE_GAUGE, DesiredBalanceMetrics.UNDESIRED_ALLOCATION_RATIO_METRIC_NAME),
+            empty()
+        );
+
+        // Collect when master
+        metrics.setNodeIsMaster(true);
+        meterRegistry.getRecorder().collect();
+        assertThat(
+            meterRegistry.getRecorder()
+                .getMeasurements(InstrumentType.LONG_GAUGE, DesiredBalanceMetrics.UNDESIRED_ALLOCATION_COUNT_METRIC_NAME)
+                .getFirst()
+                .getLong(),
+            equalTo(undesiredAllocations)
+        );
+        assertThat(
+            meterRegistry.getRecorder()
+                .getMeasurements(InstrumentType.LONG_GAUGE, DesiredBalanceMetrics.TOTAL_SHARDS_METRIC_NAME)
+                .getFirst()
+                .getLong(),
+            equalTo(totalAllocations)
+        );
+        assertThat(
+            meterRegistry.getRecorder()
+                .getMeasurements(InstrumentType.LONG_GAUGE, DesiredBalanceMetrics.UNASSIGNED_SHARDS_METRIC_NAME)
+                .getFirst()
+                .getLong(),
+            equalTo(unassignedShards)
+        );
+        assertThat(
+            meterRegistry.getRecorder()
+                .getMeasurements(InstrumentType.DOUBLE_GAUGE, DesiredBalanceMetrics.UNDESIRED_ALLOCATION_RATIO_METRIC_NAME)
+                .getFirst()
+                .getDouble(),
+            equalTo((double) undesiredAllocations / totalAllocations)
+        );
+    }
+
+    public void testUndesiredAllocationRatioIsZeroWhenTotalShardsIsZero() {
+        RecordingMeterRegistry meterRegistry = new RecordingMeterRegistry();
+        DesiredBalanceMetrics metrics = new DesiredBalanceMetrics(meterRegistry);
+        long unassignedShards = randomNonNegativeLong();
+        metrics.updateMetrics(unassignedShards, 0, 0);
+
+        metrics.setNodeIsMaster(true);
+        meterRegistry.getRecorder().collect();
+        assertThat(
+            meterRegistry.getRecorder()
+                .getMeasurements(InstrumentType.DOUBLE_GAUGE, DesiredBalanceMetrics.UNDESIRED_ALLOCATION_RATIO_METRIC_NAME)
+                .getFirst()
+                .getDouble(),
+            equalTo(0d)
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
index 1ae73c9c08137..b5f44ee9e505f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
@@ -66,7 +66,6 @@
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 import org.elasticsearch.snapshots.SnapshotsInfoService;
-import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -1215,7 +1214,7 @@ public void testRebalanceDoesNotCauseHotSpots() {
         var reconciler = new DesiredBalanceReconciler(
             clusterSettings,
             new DeterministicTaskQueue().getThreadPool(),
-            mock(MeterRegistry.class)
+            DesiredBalanceMetrics.NOOP
         );
 
         var totalOutgoingMoves = new HashMap<String, AtomicInteger>();
@@ -1297,7 +1296,7 @@ public void testShouldLogOnTooManyUndesiredAllocations() {
         final var timeInMillisSupplier = new AtomicLong();
         when(threadPool.relativeTimeInMillisSupplier()).thenReturn(timeInMillisSupplier::incrementAndGet);
 
-        var reconciler = new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, mock(MeterRegistry.class));
+        var reconciler = new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, DesiredBalanceMetrics.NOOP);
         final long initialDelayInMillis = TimeValue.timeValueMinutes(5).getMillis();
         timeInMillisSupplier.addAndGet(randomLongBetween(initialDelayInMillis, 2 * initialDelayInMillis));
 
@@ -1349,7 +1348,7 @@ public void testShouldLogOnTooManyUndesiredAllocations() {
     private static void reconcile(RoutingAllocation routingAllocation, DesiredBalance desiredBalance) {
         final var threadPool = mock(ThreadPool.class);
         when(threadPool.relativeTimeInMillisSupplier()).thenReturn(new AtomicLong()::incrementAndGet);
-        new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, mock(MeterRegistry.class)).reconcile(
+        new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, DesiredBalanceMetrics.NOOP).reconcile(
             desiredBalance,
             routingAllocation
         );
diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index ca5b9295adfd7..d8f3c4f1af48e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -70,7 +70,7 @@ public void testClusterStateSerialization() throws Exception {
             .settings(settings(IndexVersion.current()))
             .numberOfShards(10)
             .numberOfReplicas(1)
-            .eventIngestedRange(eventIngestedRangeInput, TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE);
+            .eventIngestedRange(eventIngestedRangeInput, TransportVersions.V_8_15_0);
 
         ClusterStateTestRecord result = createAndSerializeClusterState(indexMetadataBuilder, TransportVersion.current());
 
@@ -90,7 +90,7 @@ public void testClusterStateSerializationWithTimestampRangesWithOlderTransportVe
         TransportVersion versionBeforeEventIngestedInClusterState = randomFrom(
             TransportVersions.V_7_0_0,
             TransportVersions.V_8_0_0,
-            TransportVersions.ML_INFERENCE_GOOGLE_VERTEX_AI_EMBEDDINGS_ADDED  // version before EVENT_INGESTED_RANGE_IN_CLUSTER_STATE
+            TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_15_0)
         );
         {
             IndexLongFieldRange eventIngestedRangeInput = randomFrom(
diff --git a/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java b/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java
index 0cbbcdc0f1674..9ffd05a5336d6 100644
--- a/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java
@@ -77,6 +77,37 @@ public void testExponentialBackOff() {
         }
     }
 
+    public void testLinearBackoffWithLimit() {
+        long incrementMillis = randomIntBetween(10, 500);
+        long limitMillis = randomIntBetween(1000, 5000);
+        int maxNumberOfRetries = randomIntBetween(0, 30);
+        BackoffPolicy timeValues = BackoffPolicy.linearBackoff(
+            timeValueMillis(incrementMillis),
+            maxNumberOfRetries,
+            timeValueMillis(limitMillis)
+        );
+        int counter = 0;
+        for (TimeValue timeValue : timeValues) {
+            counter++;
+            long unlimitedValue = counter * incrementMillis;
+            long expectedValue = Math.min(unlimitedValue, limitMillis);
+            assertEquals(timeValueMillis(expectedValue), timeValue);
+        }
+        assertEquals(counter, maxNumberOfRetries);
+    }
+
+    public void testLinearBackoffWithoutLimit() {
+        long incrementMillis = randomIntBetween(10, 500);
+        int maxNumberOfRetries = randomIntBetween(0, 30);
+        BackoffPolicy timeValues = BackoffPolicy.linearBackoff(timeValueMillis(incrementMillis), maxNumberOfRetries, null);
+        int counter = 0;
+        for (TimeValue timeValue : timeValues) {
+            counter++;
+            assertEquals(timeValueMillis(counter * incrementMillis), timeValue);
+        }
+        assertEquals(counter, maxNumberOfRetries);
+    }
+
     public void testNoBackoff() {
         BackoffPolicy noBackoff = BackoffPolicy.noBackoff();
         int numberOfBackoffsToPerform = randomIntBetween(1, 3);
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
index 8158917f08187..9300aa992b687 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
@@ -44,7 +45,6 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.tests.analysis.MockAnalyzer;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.tests.store.MockDirectoryWrapper;
@@ -172,10 +172,10 @@ public void testPruneUnreferencedFiles() throws IOException {
         assertEquals(3, open.maxDoc());
 
         IndexSearcher s = newSearcher(open);
-        assertEquals(s.search(new TermQuery(new Term("id", "1")), 1).totalHits.value, 1);
-        assertEquals(s.search(new TermQuery(new Term("id", "2")), 1).totalHits.value, 1);
-        assertEquals(s.search(new TermQuery(new Term("id", "3")), 1).totalHits.value, 1);
-        assertEquals(s.search(new TermQuery(new Term("id", "4")), 1).totalHits.value, 0);
+        assertEquals(s.search(new TermQuery(new Term("id", "1")), 1).totalHits.value(), 1);
+        assertEquals(s.search(new TermQuery(new Term("id", "2")), 1).totalHits.value(), 1);
+        assertEquals(s.search(new TermQuery(new Term("id", "3")), 1).totalHits.value(), 1);
+        assertEquals(s.search(new TermQuery(new Term("id", "4")), 1).totalHits.value(), 0);
 
         for (String file : dir.listAll()) {
             assertFalse("unexpected file: " + file, file.equals("segments_3") || file.startsWith("_2"));
@@ -403,11 +403,6 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                     throw new UnsupportedOperationException();
                 }
 
-                @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
-                    throw new UnsupportedOperationException();
-                }
-
                 @Override
                 public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                     return new ScorerSupplier() {
@@ -464,18 +459,6 @@ public void testAsSequentialBitsUsesRandomAccess() throws IOException {
         }
     }
 
-    /**
-     * Test that the "unmap hack" is detected as supported by lucene.
-     * This works around the following bug: https://bugs.openjdk.java.net/browse/JDK-4724038
-     * <p>
-     * While not guaranteed, current status is "Critical Internal API": http://openjdk.java.net/jeps/260
-     * Additionally this checks we did not screw up the security logic around the hack.
-     */
-    public void testMMapHackSupported() throws Exception {
-        // add assume's here if needed for certain platforms, but we should know if it does not work.
-        assertTrue("MMapDirectory does not support unmapping: " + MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
-    }
-
     public void testWrapAllDocsLive() throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
@@ -508,8 +491,9 @@ public void testWrapAllDocsLive() throws Exception {
             IndexSearcher searcher = newSearcher(reader);
             Set<String> actualDocs = new HashSet<>();
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
+            StoredFields storedFields = reader.storedFields();
             for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
-                actualDocs.add(reader.document(scoreDoc.doc).get("id"));
+                actualDocs.add(storedFields.document(scoreDoc.doc).get("id"));
             }
             assertThat(actualDocs, equalTo(liveDocs));
         }
@@ -554,8 +538,9 @@ public void testWrapLiveDocsNotExposeAbortedDocuments() throws Exception {
             IndexSearcher searcher = newSearcher(reader);
             List<String> actualDocs = new ArrayList<>();
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
+            StoredFields storedFields = reader.storedFields();
             for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
-                actualDocs.add(reader.document(scoreDoc.doc).get("id"));
+                actualDocs.add(storedFields.document(scoreDoc.doc).get("id"));
             }
             assertThat(actualDocs, equalTo(liveDocs));
         }
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java b/server/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java
index 8332ff87a9d57..918dcc1bcbd42 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java
@@ -20,6 +20,7 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
@@ -127,8 +128,9 @@ public void setUp() throws Exception {
         // now go over each doc, build the relevant references and filter
         reader = DirectoryReader.open(iw);
         List<BytesRef> filterTerms = new ArrayList<>();
+        StoredFields storedFields = reader.storedFields();
         for (int docId = 0; docId < reader.maxDoc(); docId++) {
-            Document doc = reader.document(docId);
+            Document doc = storedFields.document(docId);
             addFreqs(doc, referenceAll);
             if (deletedIds.contains(doc.getField("id").stringValue()) == false) {
                 addFreqs(doc, referenceNotDeleted);
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java b/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java
index 7a8d43ebbfd18..55ca666d8588b 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/search/function/MinScoreScorerTests.java
@@ -11,15 +11,11 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.ConjunctionUtils;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.search.Weight;
 import org.apache.lucene.tests.search.AssertingScorer;
 import org.apache.lucene.tests.util.TestUtil;
 import org.elasticsearch.test.ESTestCase;
@@ -66,27 +62,8 @@ public int advance(int target) throws IOException {
         };
     }
 
-    private static Weight fakeWeight() {
-        return new Weight(new MatchAllDocsQuery()) {
-            @Override
-            public Explanation explain(LeafReaderContext context, int doc) throws IOException {
-                return null;
-            }
-
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                return null;
-            }
-
-            @Override
-            public boolean isCacheable(LeafReaderContext ctx) {
-                return false;
-            }
-        };
-    }
-
     private static Scorer hideTwoPhaseIterator(Scorer in) {
-        return new Scorer(in.getWeight()) {
+        return new Scorer() {
             @Override
             public DocIdSetIterator iterator() {
                 return TwoPhaseIterator.asDocIdSetIterator(in.twoPhaseIterator());
@@ -111,7 +88,7 @@ public int docID() {
 
     private static Scorer scorer(int maxDoc, final int[] docs, final float[] scores, final boolean twoPhase) {
         final DocIdSetIterator iterator = twoPhase ? DocIdSetIterator.all(maxDoc) : iterator(docs);
-        final Scorer scorer = new Scorer(fakeWeight()) {
+        final Scorer scorer = new Scorer() {
 
             int lastScoredDoc = -1;
             final float matchCost = (random().nextBoolean() ? 1000 : 0) + random().nextInt(2000);
@@ -192,7 +169,7 @@ public void doTestRandom(boolean twoPhase) throws IOException {
         }
         Scorer scorer = scorer(maxDoc, docs, scores, twoPhase);
         final float minScore = random().nextFloat();
-        Scorer minScoreScorer = new MinScoreScorer(fakeWeight(), scorer, minScore);
+        Scorer minScoreScorer = new MinScoreScorer(scorer, minScore);
         int doc = -1;
         while (doc != DocIdSetIterator.NO_MORE_DOCS) {
             final int target;
@@ -250,7 +227,7 @@ public void testConjunction() throws Exception {
             final float minScore;
             if (randomBoolean()) {
                 minScore = randomFloat();
-                MinScoreScorer minScoreScorer = new MinScoreScorer(scorer.getWeight(), scorer, minScore);
+                MinScoreScorer minScoreScorer = new MinScoreScorer(scorer, minScore);
                 scorers.add(minScoreScorer);
             } else {
                 scorers.add(scorer);
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java b/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java
index b1df24f4db2ad..3894efd0b7d4c 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java
@@ -117,7 +117,7 @@ public boolean incrementToken() throws IOException {
 
         final double boost10 = ((BooleanQuery) mlt.like("text", new StringReader("lucene|10 release|1"))).clauses()
             .stream()
-            .map(BooleanClause::getQuery)
+            .map(BooleanClause::query)
             .map(BoostQuery.class::cast)
             .filter(x -> ((TermQuery) x.getQuery()).getTerm().text().equals("lucene"))
             .mapToDouble(BoostQuery::getBoost)
@@ -125,7 +125,7 @@ public boolean incrementToken() throws IOException {
 
         final double boost1 = ((BooleanQuery) mlt.like("text", new StringReader("lucene|1 release|1"))).clauses()
             .stream()
-            .map(BooleanClause::getQuery)
+            .map(BooleanClause::query)
             .map(BoostQuery.class::cast)
             .filter(x -> ((TermQuery) x.getQuery()).getTerm().text().equals("lucene"))
             .mapToDouble(BoostQuery::getBoost)
@@ -178,7 +178,7 @@ public void testTopN() throws Exception {
             expectedTerms[idx++] = new Term("text", text);
         }
         for (BooleanClause clause : clauses) {
-            Term term = ((TermQuery) clause.getQuery()).getTerm();
+            Term term = ((TermQuery) clause.query()).getTerm();
             assertTrue(Arrays.asList(expectedTerms).contains(term));
         }
 
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index d182d772ee964..144dda1ad3b26 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -1122,6 +1122,57 @@ public void testSanitizeHeaders() {
         }
     }
 
+    public void testNewEmptyContext() {
+        final var threadContext = new ThreadContext(Settings.EMPTY);
+        final var header = randomBoolean() ? randomIdentifier() : randomFrom(HEADERS_TO_COPY);
+        threadContext.putHeader(header, randomIdentifier());
+
+        try (var ignored = threadContext.newEmptyContext()) {
+            assertTrue(threadContext.isDefaultContext());
+            assertNull(threadContext.getHeader(header));
+            assertTrue(threadContext.getHeaders().isEmpty());
+        }
+
+        assertNotNull(threadContext.getHeader(header));
+    }
+
+    public void testNewEmptySystemContext() {
+        final var threadContext = new ThreadContext(Settings.EMPTY);
+        final var header = randomBoolean() ? randomIdentifier() : randomFrom(HEADERS_TO_COPY);
+        threadContext.putHeader(header, randomIdentifier());
+
+        try (var ignored = threadContext.newEmptySystemContext()) {
+            assertTrue(threadContext.isSystemContext());
+            assertNull(threadContext.getHeader(header));
+            assertTrue(threadContext.getHeaders().isEmpty());
+        }
+
+        assertNotNull(threadContext.getHeader(header));
+    }
+
+    public void testRestoreExistingContext() {
+        final var threadContext = new ThreadContext(Settings.EMPTY);
+        final var header = randomIdentifier();
+        final var originalValue = randomIdentifier();
+        threadContext.putHeader(header, originalValue);
+        try (var originalContext = threadContext.newStoredContext()) {
+            assertEquals(originalValue, threadContext.getHeader(header));
+
+            try (var ignored1 = threadContext.newEmptyContext()) {
+                final var updatedValue = randomIdentifier();
+                threadContext.putHeader(header, updatedValue);
+
+                try (var ignored2 = threadContext.restoreExistingContext(originalContext)) {
+                    assertEquals(originalValue, threadContext.getHeader(header));
+                }
+
+                assertEquals(updatedValue, threadContext.getHeader(header));
+            }
+
+            assertEquals(originalValue, threadContext.getHeader(header));
+        }
+    }
+
     private String randomCase(String original) {
         int i = randomInt(original.length() - 1);
         StringBuilder sb = new StringBuilder(original);
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
index 2793e03fc3fa8..b3af430cc43e2 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
@@ -390,28 +390,31 @@ public void testText() throws Exception {
     }
 
     public void testDate() throws Exception {
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (Date) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((Date) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (Date) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((Date) null).endObject());
 
         final Date d1 = Date.from(ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timeValue(d1).endObject());
+        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timestampField("d1", d1).endObject());
+        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timestampValue(d1).endObject());
 
         final Date d2 = Date.from(ZonedDateTime.of(2016, 12, 25, 7, 59, 42, 213000000, ZoneOffset.UTC).toInstant());
-        assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().timeField("d2", d2).endObject());
-        assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().field("d2").timeValue(d2).endObject());
+        assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().timestampField("d2", d2).endObject());
+        assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().field("d2").timestampValue(d2).endObject());
     }
 
-    public void testDateField() throws Exception {
+    public void testUnixEpochMillisField() throws Exception {
         final Date d = Date.from(ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant());
 
         assertResult(
             "{'date_in_millis':1451606400000}",
-            () -> builder().startObject().timeField("date_in_millis", "date", d.getTime()).endObject()
+            () -> builder().startObject().timestampFieldsFromUnixEpochMillis("date_in_millis", "date", d.getTime()).endObject()
         );
         assertResult(
             "{'date':'2016-01-01T00:00:00.000Z','date_in_millis':1451606400000}",
-            () -> builder().humanReadable(true).startObject().timeField("date_in_millis", "date", d.getTime()).endObject()
+            () -> builder().humanReadable(true)
+                .startObject()
+                .timestampFieldsFromUnixEpochMillis("date_in_millis", "date", d.getTime())
+                .endObject()
         );
     }
 
@@ -419,7 +422,7 @@ public void testCalendar() throws Exception {
         Calendar calendar = GregorianCalendar.from(ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC));
         assertResult(
             "{'calendar':'2016-01-01T00:00:00.000Z'}",
-            () -> builder().startObject().field("calendar").timeValue(calendar).endObject()
+            () -> builder().startObject().field("calendar").timestampValue(calendar).endObject()
         );
     }
 
@@ -427,83 +430,95 @@ public void testJavaTime() throws Exception {
         final ZonedDateTime d1 = ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC);
 
         // ZonedDateTime
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (ZonedDateTime) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((ZonedDateTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (ZonedDateTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((ZonedDateTime) null).endObject());
         assertResult("{'date':null}", () -> builder().startObject().field("date", (ZonedDateTime) null).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timeValue(d1).endObject());
+        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timestampField("d1", d1).endObject());
+        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timestampValue(d1).endObject());
         assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1).endObject());
 
         // Instant
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (java.time.Instant) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((java.time.Instant) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (java.time.Instant) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((java.time.Instant) null).endObject());
         assertResult("{'date':null}", () -> builder().startObject().field("date", (java.time.Instant) null).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1.toInstant()).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timeValue(d1.toInstant()).endObject());
+        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timestampField("d1", d1.toInstant()).endObject());
+        assertResult(
+            "{'d1':'2016-01-01T00:00:00.000Z'}",
+            () -> builder().startObject().field("d1").timestampValue(d1.toInstant()).endObject()
+        );
         assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1.toInstant()).endObject());
 
         // LocalDateTime (no time zone)
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (LocalDateTime) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((LocalDateTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (LocalDateTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((LocalDateTime) null).endObject());
         assertResult("{'date':null}", () -> builder().startObject().field("date", (LocalDateTime) null).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1.toLocalDateTime()).endObject());
         assertResult(
             "{'d1':'2016-01-01T00:00:00.000Z'}",
-            () -> builder().startObject().field("d1").timeValue(d1.toLocalDateTime()).endObject()
+            () -> builder().startObject().timestampField("d1", d1.toLocalDateTime()).endObject()
+        );
+        assertResult(
+            "{'d1':'2016-01-01T00:00:00.000Z'}",
+            () -> builder().startObject().field("d1").timestampValue(d1.toLocalDateTime()).endObject()
         );
         assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1.toLocalDateTime()).endObject());
 
         // LocalDate (no time, no time zone)
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (LocalDate) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((LocalDate) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (LocalDate) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((LocalDate) null).endObject());
         assertResult("{'date':null}", () -> builder().startObject().field("date", (LocalDate) null).endObject());
-        assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().timeField("d1", d1.toLocalDate()).endObject());
-        assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().field("d1").timeValue(d1.toLocalDate()).endObject());
+        assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().timestampField("d1", d1.toLocalDate()).endObject());
+        assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().field("d1").timestampValue(d1.toLocalDate()).endObject());
         assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().field("d1", d1.toLocalDate()).endObject());
 
         // LocalTime (no date, no time zone)
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (LocalTime) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((LocalTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (LocalTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((LocalTime) null).endObject());
         assertResult("{'date':null}", () -> builder().startObject().field("date", (LocalTime) null).endObject());
-        assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().timeField("d1", d1.toLocalTime()).endObject());
-        assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().field("d1").timeValue(d1.toLocalTime()).endObject());
+        assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().timestampField("d1", d1.toLocalTime()).endObject());
+        assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().field("d1").timestampValue(d1.toLocalTime()).endObject());
         assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().field("d1", d1.toLocalTime()).endObject());
         final ZonedDateTime d2 = ZonedDateTime.of(2016, 1, 1, 7, 59, 23, 123_000_000, ZoneOffset.UTC);
-        assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().timeField("d1", d2.toLocalTime()).endObject());
-        assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().field("d1").timeValue(d2.toLocalTime()).endObject());
+        assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().timestampField("d1", d2.toLocalTime()).endObject());
+        assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().field("d1").timestampValue(d2.toLocalTime()).endObject());
         assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().field("d1", d2.toLocalTime()).endObject());
 
         // OffsetDateTime
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (OffsetDateTime) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((OffsetDateTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (OffsetDateTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((OffsetDateTime) null).endObject());
         assertResult("{'date':null}", () -> builder().startObject().field("date", (OffsetDateTime) null).endObject());
         assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1.toOffsetDateTime()).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1.toOffsetDateTime()).endObject());
         assertResult(
             "{'d1':'2016-01-01T00:00:00.000Z'}",
-            () -> builder().startObject().field("d1").timeValue(d1.toOffsetDateTime()).endObject()
+            () -> builder().startObject().timestampField("d1", d1.toOffsetDateTime()).endObject()
+        );
+        assertResult(
+            "{'d1':'2016-01-01T00:00:00.000Z'}",
+            () -> builder().startObject().field("d1").timestampValue(d1.toOffsetDateTime()).endObject()
         );
         // also test with a date that has a real offset
         OffsetDateTime offsetDateTime = d1.withZoneSameLocal(ZoneOffset.ofHours(5)).toOffsetDateTime();
         assertResult("{'d1':'2016-01-01T00:00:00.000+05:00'}", () -> builder().startObject().field("d1", offsetDateTime).endObject());
-        assertResult("{'d1':'2016-01-01T00:00:00.000+05:00'}", () -> builder().startObject().timeField("d1", offsetDateTime).endObject());
         assertResult(
             "{'d1':'2016-01-01T00:00:00.000+05:00'}",
-            () -> builder().startObject().field("d1").timeValue(offsetDateTime).endObject()
+            () -> builder().startObject().timestampField("d1", offsetDateTime).endObject()
+        );
+        assertResult(
+            "{'d1':'2016-01-01T00:00:00.000+05:00'}",
+            () -> builder().startObject().field("d1").timestampValue(offsetDateTime).endObject()
         );
 
         // OffsetTime
-        assertResult("{'date':null}", () -> builder().startObject().timeField("date", (OffsetTime) null).endObject());
-        assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((OffsetTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (OffsetTime) null).endObject());
+        assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((OffsetTime) null).endObject());
         assertResult("{'date':null}", () -> builder().startObject().field("date", (OffsetTime) null).endObject());
         final OffsetTime offsetTime = d2.toOffsetDateTime().toOffsetTime();
-        assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().timeField("o", offsetTime).endObject());
-        assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().field("o").timeValue(offsetTime).endObject());
+        assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().timestampField("o", offsetTime).endObject());
+        assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().field("o").timestampValue(offsetTime).endObject());
         assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().field("o", offsetTime).endObject());
         // also test with a date that has a real offset
         final OffsetTime zonedOffsetTime = offsetTime.withOffsetSameLocal(ZoneOffset.ofHours(5));
-        assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().timeField("o", zonedOffsetTime).endObject());
-        assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().field("o").timeValue(zonedOffsetTime).endObject());
+        assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().timestampField("o", zonedOffsetTime).endObject());
+        assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().field("o").timestampValue(zonedOffsetTime).endObject());
         assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().field("o", zonedOffsetTime).endObject());
 
         // DayOfWeek enum, not a real time value, but might be used in scripts
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
index a695fb6c45348..575382c7fb441 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
@@ -189,7 +189,7 @@ public void testDateTypesConversion() throws Exception {
         Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
         String expectedCalendar = XContentElasticsearchExtension.DEFAULT_FORMATTER.format(calendar.toInstant());
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        builder.startObject().timeField("date", date).endObject();
+        builder.startObject().timestampField("date", date).endObject();
         assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}"));
 
         builder = XContentFactory.contentBuilder(XContentType.JSON);
diff --git a/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java
index 6e8eb47035d43..e0e05c84b5649 100644
--- a/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java
+++ b/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java
@@ -84,12 +84,12 @@ public void testSimpleNumericOps() throws Exception {
             try (IndexReader reader = DirectoryReader.open(indexWriter)) {
                 IndexSearcher searcher = newSearcher(reader);
                 TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
-                Document doc = searcher.doc(topDocs.scoreDocs[0].doc);
+                Document doc = searcher.storedFields().document(topDocs.scoreDocs[0].doc);
                 IndexableField f = doc.getField("test");
                 assertThat(f.numericValue(), equalTo(2));
 
                 topDocs = searcher.search(IntPoint.newExactQuery("test", 2), 1);
-                doc = searcher.doc(topDocs.scoreDocs[0].doc);
+                doc = searcher.storedFields().document(topDocs.scoreDocs[0].doc);
                 f = doc.getField("test");
                 assertThat(f.stringValue(), equalTo("2"));
             }
@@ -115,7 +115,7 @@ public void testOrdering() throws Exception {
                 IndexSearcher searcher = newSearcher(reader);
                 TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
                 final ArrayList<String> fieldsOrder = new ArrayList<>();
-                searcher.doc(topDocs.scoreDocs[0].doc, new StoredFieldVisitor() {
+                searcher.storedFields().document(topDocs.scoreDocs[0].doc, new StoredFieldVisitor() {
                     @Override
                     public Status needsField(FieldInfo fieldInfo) throws IOException {
                         fieldsOrder.add(fieldInfo.name);
diff --git a/server/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java
index 97a6faaa5c6f6..01c4ac3c6fd6a 100644
--- a/server/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java
+++ b/server/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java
@@ -54,7 +54,7 @@ public void testVectorHighlighter() throws Exception {
         IndexSearcher searcher = newSearcher(reader);
         TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
 
-        assertThat(topDocs.totalHits.value, equalTo(1L));
+        assertThat(topDocs.totalHits.value(), equalTo(1L));
 
         FastVectorHighlighter highlighter = new FastVectorHighlighter();
         String fragment = highlighter.getBestFragment(
@@ -87,7 +87,7 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
         IndexReader reader = searcher.getIndexReader();
         TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
 
-        assertThat(topDocs.totalHits.value, equalTo(1L));
+        assertThat(topDocs.totalHits.value(), equalTo(1L));
 
         FastVectorHighlighter highlighter = new FastVectorHighlighter();
 
@@ -143,7 +143,7 @@ public void testVectorHighlighterNoStore() throws Exception {
         IndexSearcher searcher = newSearcher(reader);
         TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
 
-        assertThat(topDocs.totalHits.value, equalTo(1L));
+        assertThat(topDocs.totalHits.value(), equalTo(1L));
 
         FastVectorHighlighter highlighter = new FastVectorHighlighter();
         String fragment = highlighter.getBestFragment(
@@ -170,7 +170,7 @@ public void testVectorHighlighterNoTermVector() throws Exception {
         IndexSearcher searcher = newSearcher(reader);
         TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
 
-        assertThat(topDocs.totalHits.value, equalTo(1L));
+        assertThat(topDocs.totalHits.value(), equalTo(1L));
 
         FastVectorHighlighter highlighter = new FastVectorHighlighter();
         String fragment = highlighter.getBestFragment(
diff --git a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java
index 8bfd4c7c5ac68..22308e15f4845 100644
--- a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java
@@ -167,10 +167,6 @@ public void testUpgradeMarksPreviousVersion() {
         assertThat(nodeMetadata.previousNodeVersion(), equalTo(buildVersion));
     }
 
-    public static Version tooNewVersion() {
-        return Version.fromId(between(Version.CURRENT.id + 1, 99999999));
-    }
-
     public static IndexVersion tooNewIndexVersion() {
         return IndexVersion.fromId(between(IndexVersion.current().id() + 1, 99999999));
     }
@@ -179,10 +175,6 @@ public static BuildVersion tooNewBuildVersion() {
         return BuildVersion.fromVersionId(between(Version.CURRENT.id() + 1, 99999999));
     }
 
-    public static Version tooOldVersion() {
-        return Version.fromId(between(1, Version.CURRENT.minimumCompatibilityVersion().id - 1));
-    }
-
     public static BuildVersion tooOldBuildVersion() {
         return BuildVersion.fromVersionId(between(1, Version.CURRENT.minimumCompatibilityVersion().id - 1));
     }
diff --git a/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java b/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java
index bf3fc1697aa44..c7614e2d98eed 100644
--- a/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java
+++ b/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java
@@ -96,7 +96,9 @@ public void testFailsOnEmptyPath() {
     }
 
     public void testFailsIfUnnecessary() throws IOException {
-        final Version nodeVersion = Version.fromId(between(Version.CURRENT.minimumCompatibilityVersion().id, Version.CURRENT.id));
+        final BuildVersion nodeVersion = BuildVersion.fromVersionId(
+            between(Version.CURRENT.minimumCompatibilityVersion().id, Version.CURRENT.id)
+        );
         PersistedClusterStateService.overrideVersion(nodeVersion, dataPaths);
         final MockTerminal mockTerminal = MockTerminal.create();
         final ElasticsearchException elasticsearchException = expectThrows(
@@ -107,7 +109,7 @@ public void testFailsIfUnnecessary() throws IOException {
             elasticsearchException.getMessage(),
             allOf(
                 containsString("compatible with current version"),
-                containsString(Version.CURRENT.toString()),
+                containsString(BuildVersion.current().toString()),
                 containsString(nodeVersion.toString())
             )
         );
@@ -115,7 +117,7 @@ public void testFailsIfUnnecessary() throws IOException {
     }
 
     public void testWarnsIfTooOld() throws Exception {
-        final Version nodeVersion = NodeMetadataTests.tooOldVersion();
+        final BuildVersion nodeVersion = NodeMetadataTests.tooOldBuildVersion();
         PersistedClusterStateService.overrideVersion(nodeVersion, dataPaths);
         final MockTerminal mockTerminal = MockTerminal.create();
         mockTerminal.addTextInput("n");
@@ -137,11 +139,11 @@ public void testWarnsIfTooOld() throws Exception {
         expectThrows(IllegalStateException.class, () -> mockTerminal.readText(""));
 
         final NodeMetadata nodeMetadata = PersistedClusterStateService.nodeMetadata(dataPaths);
-        assertThat(nodeMetadata.nodeVersion().toVersion(), equalTo(nodeVersion));
+        assertThat(nodeMetadata.nodeVersion(), equalTo(nodeVersion));
     }
 
     public void testWarnsIfTooNew() throws Exception {
-        final Version nodeVersion = NodeMetadataTests.tooNewVersion();
+        final BuildVersion nodeVersion = NodeMetadataTests.tooNewBuildVersion();
         PersistedClusterStateService.overrideVersion(nodeVersion, dataPaths);
         final MockTerminal mockTerminal = MockTerminal.create();
         mockTerminal.addTextInput(randomFrom("yy", "Yy", "n", "yes", "true", "N", "no"));
@@ -162,11 +164,11 @@ public void testWarnsIfTooNew() throws Exception {
         expectThrows(IllegalStateException.class, () -> mockTerminal.readText(""));
 
         final NodeMetadata nodeMetadata = PersistedClusterStateService.nodeMetadata(dataPaths);
-        assertThat(nodeMetadata.nodeVersion().toVersion(), equalTo(nodeVersion));
+        assertThat(nodeMetadata.nodeVersion(), equalTo(nodeVersion));
     }
 
     public void testOverwritesIfTooOld() throws Exception {
-        final Version nodeVersion = NodeMetadataTests.tooOldVersion();
+        final BuildVersion nodeVersion = NodeMetadataTests.tooOldBuildVersion();
         PersistedClusterStateService.overrideVersion(nodeVersion, dataPaths);
         final MockTerminal mockTerminal = MockTerminal.create();
         mockTerminal.addTextInput(randomFrom("y", "Y"));
@@ -189,7 +191,7 @@ public void testOverwritesIfTooOld() throws Exception {
     }
 
     public void testOverwritesIfTooNew() throws Exception {
-        final Version nodeVersion = NodeMetadataTests.tooNewVersion();
+        final BuildVersion nodeVersion = NodeMetadataTests.tooNewBuildVersion();
         PersistedClusterStateService.overrideVersion(nodeVersion, dataPaths);
         final MockTerminal mockTerminal = MockTerminal.create();
         mockTerminal.addTextInput(randomFrom("y", "Y"));
diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
index 40c4e064216f1..1bbab8bf782bd 100644
--- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
@@ -9,21 +9,15 @@
 package org.elasticsearch.gateway;
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-import java.util.HashMap;
-
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.nullValue;
 
 public class MetaStateServiceTests extends ESTestCase {
@@ -75,130 +69,4 @@ public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
         assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metadata.persistentSettings()));
         assertThat(metaStateService.loadGlobalState().hasIndex("test1"), equalTo(false));
     }
-
-    public void testLoadFullStateBWC() throws Exception {
-        IndexMetadata indexMetadata = indexMetadata("test1");
-        Metadata metadata = Metadata.builder()
-            .persistentSettings(Settings.builder().put("test1", "value1").build())
-            .put(indexMetadata, true)
-            .build();
-
-        long globalGeneration = MetaStateWriterUtils.writeGlobalState(env, "test_write", metadata);
-        long indexGeneration = MetaStateWriterUtils.writeIndex(env, "test_write", indexMetadata);
-
-        Tuple<Manifest, Metadata> manifestAndMetadata = metaStateService.loadFullState();
-        Manifest manifest = manifestAndMetadata.v1();
-        assertThat(manifest.globalGeneration(), equalTo(globalGeneration));
-        assertThat(manifest.indexGenerations(), hasKey(indexMetadata.getIndex()));
-        assertThat(manifest.indexGenerations().get(indexMetadata.getIndex()), equalTo(indexGeneration));
-
-        Metadata loadedMetadata = manifestAndMetadata.v2();
-        assertThat(loadedMetadata.persistentSettings(), equalTo(metadata.persistentSettings()));
-        assertThat(loadedMetadata.hasIndex("test1"), equalTo(true));
-        assertThat(loadedMetadata.index("test1"), equalTo(indexMetadata));
-    }
-
-    public void testLoadEmptyStateNoManifest() throws IOException {
-        Tuple<Manifest, Metadata> manifestAndMetadata = metaStateService.loadFullState();
-
-        Manifest manifest = manifestAndMetadata.v1();
-        assertTrue(manifest.isEmpty());
-
-        Metadata metadata = manifestAndMetadata.v2();
-        assertTrue(Metadata.isGlobalStateEquals(metadata, Metadata.EMPTY_METADATA));
-    }
-
-    public void testLoadEmptyStateWithManifest() throws IOException {
-        Manifest manifest = Manifest.empty();
-        MetaStateWriterUtils.writeManifestAndCleanup(env, "test", manifest);
-
-        Tuple<Manifest, Metadata> manifestAndMetadata = metaStateService.loadFullState();
-        assertTrue(manifestAndMetadata.v1().isEmpty());
-        Metadata metadata = manifestAndMetadata.v2();
-        assertTrue(Metadata.isGlobalStateEquals(metadata, Metadata.EMPTY_METADATA));
-    }
-
-    public void testLoadFullStateMissingGlobalMetadata() throws IOException {
-        IndexMetadata index = indexMetadata("test1");
-        long indexGeneration = MetaStateWriterUtils.writeIndex(env, "test", index);
-        Manifest manifest = new Manifest(
-            randomNonNegativeLong(),
-            randomNonNegativeLong(),
-            Manifest.empty().globalGeneration(),
-            new HashMap<Index, Long>() {
-                {
-                    put(index.getIndex(), indexGeneration);
-                }
-            }
-        );
-        assertTrue(manifest.isGlobalGenerationMissing());
-        MetaStateWriterUtils.writeManifestAndCleanup(env, "test", manifest);
-
-        Tuple<Manifest, Metadata> manifestAndMetadata = metaStateService.loadFullState();
-        assertThat(manifestAndMetadata.v1(), equalTo(manifest));
-        Metadata loadedMetadata = manifestAndMetadata.v2();
-        assertTrue(Metadata.isGlobalStateEquals(loadedMetadata, Metadata.EMPTY_METADATA));
-        assertThat(loadedMetadata.hasIndex("test1"), equalTo(true));
-        assertThat(loadedMetadata.index("test1"), equalTo(index));
-    }
-
-    public void testLoadFullStateAndUpdateAndClean() throws IOException {
-        IndexMetadata index = indexMetadata("test1");
-        Metadata metadata = Metadata.builder()
-            .persistentSettings(Settings.builder().put("test1", "value1").build())
-            .put(index, true)
-            .build();
-
-        long globalGeneration = MetaStateWriterUtils.writeGlobalState(env, "first global state write", metadata);
-        long indexGeneration = MetaStateWriterUtils.writeIndex(env, "first index state write", index);
-
-        Manifest manifest = new Manifest(randomNonNegativeLong(), randomNonNegativeLong(), globalGeneration, new HashMap<Index, Long>() {
-            {
-                put(index.getIndex(), indexGeneration);
-            }
-        });
-        MetaStateWriterUtils.writeManifestAndCleanup(env, "first manifest write", manifest);
-
-        Metadata newMetadata = Metadata.builder()
-            .persistentSettings(Settings.builder().put("test1", "value2").build())
-            .put(index, true)
-            .build();
-        globalGeneration = MetaStateWriterUtils.writeGlobalState(env, "second global state write", newMetadata);
-
-        Tuple<Manifest, Metadata> manifestAndMetadata = metaStateService.loadFullState();
-        assertThat(manifestAndMetadata.v1(), equalTo(manifest));
-
-        Metadata loadedMetadata = manifestAndMetadata.v2();
-        assertThat(loadedMetadata.persistentSettings(), equalTo(metadata.persistentSettings()));
-        assertThat(loadedMetadata.hasIndex("test1"), equalTo(true));
-        assertThat(loadedMetadata.index("test1"), equalTo(index));
-
-        manifest = new Manifest(randomNonNegativeLong(), randomNonNegativeLong(), globalGeneration, new HashMap<Index, Long>() {
-            {
-                put(index.getIndex(), indexGeneration);
-            }
-        });
-
-        MetaStateWriterUtils.writeManifestAndCleanup(env, "second manifest write", manifest);
-        Metadata.FORMAT.cleanupOldFiles(globalGeneration, env.nodeDataPaths());
-        IndexMetadata.FORMAT.cleanupOldFiles(indexGeneration, env.indexPaths(index.getIndex()));
-
-        manifestAndMetadata = metaStateService.loadFullState();
-        assertThat(manifestAndMetadata.v1(), equalTo(manifest));
-
-        loadedMetadata = manifestAndMetadata.v2();
-        assertThat(loadedMetadata.persistentSettings(), equalTo(newMetadata.persistentSettings()));
-        assertThat(loadedMetadata.hasIndex("test1"), equalTo(true));
-        assertThat(loadedMetadata.index("test1"), equalTo(index));
-
-        if (randomBoolean()) {
-            metaStateService.unreferenceAll();
-        } else {
-            metaStateService.deleteAll();
-        }
-        manifestAndMetadata = metaStateService.loadFullState();
-        assertTrue(manifestAndMetadata.v1().isEmpty());
-        metadata = manifestAndMetadata.v2();
-        assertTrue(Metadata.isGlobalStateEquals(metadata, Metadata.EMPTY_METADATA));
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java
index 022a6994496ae..9419c63f9c48a 100644
--- a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java
@@ -227,7 +227,7 @@ public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOExcep
             }
             long checksumAfterCorruption;
             long actualChecksumAfterCorruption;
-            try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
+            try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString())) {
                 assertThat(input.getFilePointer(), is(0L));
                 input.seek(input.length() - 8); // one long is the checksum... 8 bytes
                 checksumAfterCorruption = input.getChecksum();
diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
index db656b1fc5a94..4428a7e078510 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
@@ -20,6 +20,7 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
@@ -53,6 +54,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -1413,14 +1415,17 @@ public void testOverrideLuceneVersion() throws IOException {
                 assertThat(clusterState.metadata().version(), equalTo(version));
 
             }
+            @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS)
+            BuildVersion overrideVersion = BuildVersion.fromVersionId(Version.V_8_0_0.id);
+
             NodeMetadata prevMetadata = PersistedClusterStateService.nodeMetadata(persistedClusterStateService.getDataPaths());
             assertEquals(BuildVersion.current(), prevMetadata.nodeVersion());
-            PersistedClusterStateService.overrideVersion(Version.V_8_0_0, persistedClusterStateService.getDataPaths());
+            PersistedClusterStateService.overrideVersion(overrideVersion, persistedClusterStateService.getDataPaths());
             NodeMetadata metadata = PersistedClusterStateService.nodeMetadata(persistedClusterStateService.getDataPaths());
-            assertEquals(BuildVersion.fromVersionId(Version.V_8_0_0.id()), metadata.nodeVersion());
+            assertEquals(overrideVersion, metadata.nodeVersion());
             for (Path p : persistedClusterStateService.getDataPaths()) {
                 NodeMetadata individualMetadata = PersistedClusterStateService.nodeMetadata(p);
-                assertEquals(BuildVersion.fromVersionId(Version.V_8_0_0.id()), individualMetadata.nodeVersion());
+                assertEquals(overrideVersion, individualMetadata.nodeVersion());
             }
         }
     }
@@ -1776,9 +1781,10 @@ private static void forEachDocument(DirectoryReader reader, Set<String> types, C
                     final Bits liveDocs = leafReaderContext.reader().getLiveDocs();
                     final IntPredicate isLiveDoc = liveDocs == null ? i -> true : liveDocs::get;
                     final DocIdSetIterator docIdSetIterator = scorer.iterator();
+                    StoredFields storedFields = leafReaderContext.reader().storedFields();
                     while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
                         if (isLiveDoc.test(docIdSetIterator.docID())) {
-                            final Document document = leafReaderContext.reader().document(docIdSetIterator.docID());
+                            final Document document = storedFields.document(docIdSetIterator.docID());
                             document.add(new StringField(TYPE_FIELD_NAME, typeName, Field.Store.NO));
                             consumer.accept(document);
                         }
diff --git a/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java b/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
index 91936ea8b9092..cba0dacccd8bd 100644
--- a/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
+++ b/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
@@ -198,8 +198,8 @@ public void testChunkCount() {
         diagnosisList.add(diagnosis2);
         HealthIndicatorResult result = new HealthIndicatorResult(name, status, symptom, details, impacts, diagnosisList);
 
-        // -> each Diagnosis yields 5 chunks => 10 chunks from both diagnosis
-        // -> HealthIndicatorResult surrounds the diagnosis list by 2 chunks
-        AbstractChunkedSerializingTestCase.assertChunkCount(result, ignored -> 12);
+        // -> each Diagnosis yields 6 chunks => 12 chunks from both diagnosis
+        // -> HealthIndicatorResult surrounds the diagnosis list by 5 chunks
+        AbstractChunkedSerializingTestCase.assertChunkCount(result, ignored -> (6 * 2) + 5);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
index 7b264ac93511b..532a2ff024e8f 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
@@ -290,7 +290,7 @@ public void testRefreshActuallyWorks() throws Exception {
             // we are running on updateMetadata if the interval changes
             try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
                 TopDocs search = searcher.search(new MatchAllDocsQuery(), 10);
-                assertEquals(1, search.totalHits.value);
+                assertEquals(1, search.totalHits.value());
             }
         });
         assertFalse(refreshTask.isClosed());
@@ -304,7 +304,7 @@ public void testRefreshActuallyWorks() throws Exception {
             // this one becomes visible due to the force refresh we are running on updateMetadata if the interval changes
             try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
                 TopDocs search = searcher.search(new MatchAllDocsQuery(), 10);
-                assertEquals(2, search.totalHits.value);
+                assertEquals(2, search.totalHits.value());
             }
         });
         prepareIndex("test").setId("2").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
@@ -312,7 +312,7 @@ public void testRefreshActuallyWorks() throws Exception {
             // this one becomes visible due to the scheduled refresh
             try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
                 TopDocs search = searcher.search(new MatchAllDocsQuery(), 10);
-                assertEquals(3, search.totalHits.value);
+                assertEquals(3, search.totalHits.value());
             }
         });
     }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java
index 387340c0a6f50..adac8bf204f3e 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java
@@ -23,15 +23,24 @@
 public class IndexSettingProviderTests extends ESSingleNodeTestCase {
 
     public void testIndexCreation() throws Exception {
-        var indexService = createIndex("my-index1");
+        Settings settings = Settings.builder().put("index.mapping.depth.limit", 10).build();
+        var indexService = createIndex("my-index1", settings);
         assertFalse(indexService.getIndexSettings().getSettings().hasValue("index.refresh_interval"));
+        assertEquals("10", indexService.getIndexSettings().getSettings().get("index.mapping.depth.limit"));
 
         INDEX_SETTING_PROVIDER1_ENABLED.set(true);
-        indexService = createIndex("my-index2");
+        indexService = createIndex("my-index2", settings);
         assertTrue(indexService.getIndexSettings().getSettings().hasValue("index.refresh_interval"));
+        assertEquals("10", indexService.getIndexSettings().getSettings().get("index.mapping.depth.limit"));
 
+        INDEX_SETTING_OVERRULING.set(true);
+        indexService = createIndex("my-index3", settings);
+        assertTrue(indexService.getIndexSettings().getSettings().hasValue("index.refresh_interval"));
+        assertEquals("100", indexService.getIndexSettings().getSettings().get("index.mapping.depth.limit"));
+
+        INDEX_SETTING_DEPTH_ENABLED.set(false);
         INDEX_SETTING_PROVIDER2_ENABLED.set(true);
-        var e = expectThrows(IllegalArgumentException.class, () -> createIndex("my-index3"));
+        var e = expectThrows(IllegalArgumentException.class, () -> createIndex("my-index4", settings));
         assertEquals(
             "additional index setting [index.refresh_interval] added by [TestIndexSettingsProvider] is already present",
             e.getMessage()
@@ -47,7 +56,7 @@ public static class Plugin1 extends Plugin {
 
         @Override
         public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) {
-            return List.of(new TestIndexSettingsProvider("index.refresh_interval", "-1", INDEX_SETTING_PROVIDER1_ENABLED));
+            return List.of(new TestIndexSettingsProvider("-1", INDEX_SETTING_PROVIDER1_ENABLED));
         }
 
     }
@@ -56,22 +65,22 @@ public static class Plugin2 extends Plugin {
 
         @Override
         public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) {
-            return List.of(new TestIndexSettingsProvider("index.refresh_interval", "100s", INDEX_SETTING_PROVIDER2_ENABLED));
+            return List.of(new TestIndexSettingsProvider("100s", INDEX_SETTING_PROVIDER2_ENABLED));
         }
     }
 
     private static final AtomicBoolean INDEX_SETTING_PROVIDER1_ENABLED = new AtomicBoolean(false);
     private static final AtomicBoolean INDEX_SETTING_PROVIDER2_ENABLED = new AtomicBoolean(false);
+    private static final AtomicBoolean INDEX_SETTING_DEPTH_ENABLED = new AtomicBoolean(true);
+    private static final AtomicBoolean INDEX_SETTING_OVERRULING = new AtomicBoolean(false);
 
     static class TestIndexSettingsProvider implements IndexSettingProvider {
 
-        private final String settingName;
-        private final String settingValue;
+        private final String intervalValue;
         private final AtomicBoolean enabled;
 
-        TestIndexSettingsProvider(String settingName, String settingValue, AtomicBoolean enabled) {
-            this.settingName = settingName;
-            this.settingValue = settingValue;
+        TestIndexSettingsProvider(String intervalValue, AtomicBoolean enabled) {
+            this.intervalValue = intervalValue;
             this.enabled = enabled;
         }
 
@@ -79,17 +88,26 @@ static class TestIndexSettingsProvider implements IndexSettingProvider {
         public Settings getAdditionalIndexSettings(
             String indexName,
             String dataStreamName,
-            boolean isTimeSeries,
+            IndexMode templateIndexMode,
             Metadata metadata,
             Instant resolvedAt,
             Settings indexTemplateAndCreateRequestSettings,
             List<CompressedXContent> combinedTemplateMappings
         ) {
             if (enabled.get()) {
-                return Settings.builder().put(settingName, settingValue).build();
+                var builder = Settings.builder().put("index.refresh_interval", intervalValue);
+                if (INDEX_SETTING_DEPTH_ENABLED.get()) {
+                    builder.put("index.mapping.depth.limit", 100);
+                }
+                return builder.build();
             } else {
                 return Settings.EMPTY;
             }
         }
+
+        @Override
+        public boolean overrulesTemplateAndRequestSettings() {
+            return INDEX_SETTING_OVERRULING.get();
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
index 8a66bb1464a5b..23fc788a89bde 100644
--- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
@@ -13,14 +13,24 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class LogsIndexModeTests extends ESTestCase {
     public void testLogsIndexModeSetting() {
         assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGSDB));
     }
 
-    public void testSortField() {
+    public void testDefaultHostNameSortField() {
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
+        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true));
+        assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name"));
+    }
+
+    public void testCustomSortField() {
         final Settings sortSettings = Settings.builder()
             .put(buildSettings())
             .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
@@ -29,7 +39,9 @@ public void testSortField() {
         assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
         assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB));
-        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()), equalTo("agent_id"));
+        assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(false));
+        assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), not(containsString("host")));
     }
 
     public void testSortMode() {
diff --git a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java
index b12cd256eebcc..d7d5c886e0741 100644
--- a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java
@@ -48,6 +48,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.cluster.node.DiscoveryNode.STATELESS_ENABLED_SETTING_NAME;
+import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
 import static org.elasticsearch.index.cache.bitset.BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -272,21 +273,35 @@ public void testShouldLoadRandomAccessFiltersEagerly() {
         for (var hasIndexRole : values) {
             for (var loadFiltersEagerly : values) {
                 for (var isStateless : values) {
-                    boolean result = BitsetFilterCache.shouldLoadRandomAccessFiltersEagerly(
-                        bitsetFilterCacheSettings(isStateless, hasIndexRole, loadFiltersEagerly)
-                    );
-                    if (isStateless) {
-                        assertEquals(loadFiltersEagerly && hasIndexRole == false, result);
-                    } else {
-                        assertEquals(loadFiltersEagerly, result);
+                    for (var fastRefresh : values) {
+                        if (isStateless == false && fastRefresh) {
+                            // fast refresh is only relevant for stateless indices
+                            continue;
+                        }
+
+                        boolean result = BitsetFilterCache.shouldLoadRandomAccessFiltersEagerly(
+                            bitsetFilterCacheSettings(isStateless, hasIndexRole, loadFiltersEagerly, fastRefresh)
+                        );
+                        if (isStateless) {
+                            assertEquals(loadFiltersEagerly && ((hasIndexRole && fastRefresh) || hasIndexRole == false), result);
+                        } else {
+                            assertEquals(loadFiltersEagerly, result);
+                        }
                     }
                 }
             }
         }
     }
 
-    private IndexSettings bitsetFilterCacheSettings(boolean isStateless, boolean hasIndexRole, boolean loadFiltersEagerly) {
-        var indexSettingsBuilder = Settings.builder().put(INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING.getKey(), loadFiltersEagerly);
+    private IndexSettings bitsetFilterCacheSettings(
+        boolean isStateless,
+        boolean hasIndexRole,
+        boolean loadFiltersEagerly,
+        boolean fastRefresh
+    ) {
+        var indexSettingsBuilder = Settings.builder()
+            .put(INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING.getKey(), loadFiltersEagerly)
+            .put(INDEX_FAST_REFRESH_SETTING.getKey(), fastRefresh);
 
         var nodeSettingsBuilder = Settings.builder()
             .putList(
diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index 10b0b54d2d7e2..9e4a19eb039fd 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -52,7 +52,7 @@ public void testResolveDefaultCodecs() throws Exception {
         assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
         CodecService codecService = createCodecService();
         assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
-        assertThat(codecService.codec("default"), instanceOf(Elasticsearch816Codec.class));
+        assertThat(codecService.codec("default"), instanceOf(Elasticsearch900Codec.class));
     }
 
     public void testDefault() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java
index 7da5463ea46ff..62474113d73d2 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java
@@ -31,17 +31,18 @@
 import java.util.Random;
 
 public class DocValuesForUtilTests extends LuceneTestCase {
+    int NUMERIC_BLOCK_SIZE = 1 << 7;
 
     public void testEncodeDecode() throws IOException {
         final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
-        final long[] values = new long[iterations * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
+        final long[] values = new long[iterations * NUMERIC_BLOCK_SIZE];
         final int[] bpvs = new int[iterations];
 
         for (int i = 0; i < iterations; ++i) {
             final int bpv = TestUtil.nextInt(random(), 1, 64);
             bpvs[i] = DocValuesForUtil.roundBits(bpv);
-            for (int j = 0; j < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++j) {
-                values[i * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE + j] = bpv == 64
+            for (int j = 0; j < NUMERIC_BLOCK_SIZE; ++j) {
+                values[i * NUMERIC_BLOCK_SIZE + j] = bpv == 64
                     ? random().nextLong()
                     : TestUtil.nextLong(random(), 0, PackedInts.maxValue(bpv));
             }
@@ -53,12 +54,12 @@ public void testEncodeDecode() throws IOException {
         {
             // encode
             IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
-            final DocValuesForUtil forUtil = new DocValuesForUtil();
+            final DocValuesForUtil forUtil = new DocValuesForUtil(NUMERIC_BLOCK_SIZE);
 
             for (int i = 0; i < iterations; ++i) {
-                long[] source = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
-                for (int j = 0; j < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++j) {
-                    source[j] = values[i * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE + j];
+                long[] source = new long[NUMERIC_BLOCK_SIZE];
+                for (int j = 0; j < NUMERIC_BLOCK_SIZE; ++j) {
+                    source[j] = values[i * NUMERIC_BLOCK_SIZE + j];
                 }
                 out.writeByte((byte) bpvs[i]);
                 forUtil.encode(source, bpvs[i], out);
@@ -70,17 +71,14 @@ public void testEncodeDecode() throws IOException {
         {
             // decode
             IndexInput in = d.openInput("test.bin", IOContext.READONCE);
-            final long[] restored = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
+            final DocValuesForUtil forUtil = new DocValuesForUtil(NUMERIC_BLOCK_SIZE);
+            final long[] restored = new long[NUMERIC_BLOCK_SIZE];
             for (int i = 0; i < iterations; ++i) {
                 final int bitsPerValue = in.readByte();
-                DocValuesForUtil.decode(bitsPerValue, in, restored);
+                forUtil.decode(bitsPerValue, in, restored);
                 assertArrayEquals(
                     Arrays.toString(restored),
-                    ArrayUtil.copyOfSubArray(
-                        values,
-                        i * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE,
-                        (i + 1) * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE
-                    ),
+                    ArrayUtil.copyOfSubArray(values, i * NUMERIC_BLOCK_SIZE, (i + 1) * NUMERIC_BLOCK_SIZE),
                     restored
                 );
             }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java
index 288830276915e..0010c25179b69 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java
@@ -23,11 +23,11 @@
 
 public class ES87TSDBDocValuesEncoderTests extends LuceneTestCase {
 
-    private final ES87TSDBDocValuesEncoder encoder;
+    private final TSDBDocValuesEncoder encoder;
     private final int blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
 
     public ES87TSDBDocValuesEncoderTests() {
-        this.encoder = new ES87TSDBDocValuesEncoder();
+        this.encoder = new TSDBDocValuesEncoder(blockSize);
     }
 
     public void testRandomValues() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java
index d43a1e09d71a3..12a17f5c263a8 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java
@@ -115,7 +115,6 @@ public void testSortedSetDocValuesSingleUniqueValue() throws IOException {
                     assertEquals(0, field.nextOrd());
                     BytesRef scratch = field.lookupOrd(0);
                     assertEquals("value", scratch.utf8ToString());
-                    assertEquals(SortedSetDocValues.NO_MORE_ORDS, field.nextOrd());
                 }
                 assertEquals(DocIdSetIterator.NO_MORE_DOCS, field.nextDoc());
                 for (int i = 0; i < NUM_DOCS; i++) {
@@ -126,7 +125,6 @@ public void testSortedSetDocValuesSingleUniqueValue() throws IOException {
                     BytesRef scratch = fieldN.lookupOrd(0);
                     assertEquals("value" + i, scratch.utf8ToString());
                     assertEquals(DocIdSetIterator.NO_MORE_DOCS, fieldN.nextDoc());
-                    assertEquals(SortedSetDocValues.NO_MORE_ORDS, fieldN.nextOrd());
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatVariableSkipIntervalTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatVariableSkipIntervalTests.java
new file mode 100644
index 0000000000000..099b59808ef4a
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatVariableSkipIntervalTests.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.TestUtil;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/** Tests ES87TSDBDocValuesFormat with custom skipper interval size. */
+public class ES87TSDBDocValuesFormatVariableSkipIntervalTests extends BaseDocValuesFormatTestCase {
+
+    @Override
+    protected Codec getCodec() {
+        // small interval size to test with many intervals
+        return TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat(random().nextInt(4, 16)));
+    }
+
+    public void testSkipIndexIntervalSize() {
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> new ES87TSDBDocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2))
+        );
+        assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1"));
+    }
+
+    public void testSkipperAllEqualValue() throws IOException {
+        final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+        try (Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+            final int numDocs = atLeast(100);
+            for (int i = 0; i < numDocs; i++) {
+                final Document doc = new Document();
+                doc.add(NumericDocValuesField.indexedField("dv", 0L));
+                writer.addDocument(doc);
+            }
+            writer.forceMerge(1);
+            try (IndexReader reader = writer.getReader()) {
+                assertEquals(1, reader.leaves().size());
+                final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+                assertNotNull(skipper);
+                skipper.advance(0);
+                assertEquals(0L, skipper.minValue(0));
+                assertEquals(0L, skipper.maxValue(0));
+                assertEquals(numDocs, skipper.docCount(0));
+                skipper.advance(skipper.maxDocID(0) + 1);
+                assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+            }
+        }
+    }
+
+    // break on different value
+    public void testSkipperFewValuesSorted() throws IOException {
+        final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+        boolean reverse = random().nextBoolean();
+        config.setIndexSort(new Sort(new SortField("dv", SortField.Type.LONG, reverse)));
+        try (Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+            final int intervals = random().nextInt(2, 10);
+            final int[] numDocs = new int[intervals];
+            for (int i = 0; i < intervals; i++) {
+                numDocs[i] = random().nextInt(10) + 16;
+                for (int j = 0; j < numDocs[i]; j++) {
+                    final Document doc = new Document();
+                    doc.add(NumericDocValuesField.indexedField("dv", i));
+                    writer.addDocument(doc);
+                }
+            }
+            writer.forceMerge(1);
+            try (IndexReader reader = writer.getReader()) {
+                assertEquals(1, reader.leaves().size());
+                final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+                assertNotNull(skipper);
+                assertEquals(Arrays.stream(numDocs).sum(), skipper.docCount());
+                skipper.advance(0);
+                if (reverse) {
+                    for (int i = intervals - 1; i >= 0; i--) {
+                        assertEquals(i, skipper.minValue(0));
+                        assertEquals(i, skipper.maxValue(0));
+                        assertEquals(numDocs[i], skipper.docCount(0));
+                        skipper.advance(skipper.maxDocID(0) + 1);
+                    }
+                } else {
+                    for (int i = 0; i < intervals; i++) {
+                        assertEquals(i, skipper.minValue(0));
+                        assertEquals(i, skipper.maxValue(0));
+                        assertEquals(numDocs[i], skipper.docCount(0));
+                        skipper.advance(skipper.maxDocID(0) + 1);
+                    }
+                }
+                assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+            }
+        }
+    }
+
+    // break on empty doc values
+    public void testSkipperAllEqualValueWithGaps() throws IOException {
+        final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+        config.setIndexSort(new Sort(new SortField("sort", SortField.Type.LONG, false)));
+        try (Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+            final int gaps = random().nextInt(2, 10);
+            final int[] numDocs = new int[gaps];
+            long totaldocs = 0;
+            for (int i = 0; i < gaps; i++) {
+                numDocs[i] = random().nextInt(10) + 16;
+                for (int j = 0; j < numDocs[i]; j++) {
+                    final Document doc = new Document();
+                    doc.add(new NumericDocValuesField("sort", totaldocs++));
+                    doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+                    writer.addDocument(doc);
+                }
+                // add doc with empty "dv"
+                final Document doc = new Document();
+                doc.add(new NumericDocValuesField("sort", totaldocs++));
+                writer.addDocument(doc);
+            }
+            writer.forceMerge(1);
+            try (IndexReader reader = writer.getReader()) {
+                assertEquals(1, reader.leaves().size());
+                final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+                assertNotNull(skipper);
+                assertEquals(Arrays.stream(numDocs).sum(), skipper.docCount());
+                skipper.advance(0);
+                for (int i = 0; i < gaps; i++) {
+                    assertEquals(0L, skipper.minValue(0));
+                    assertEquals(0L, skipper.maxValue(0));
+                    assertEquals(numDocs[i], skipper.docCount(0));
+                    skipper.advance(skipper.maxDocID(0) + 1);
+                }
+                assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+            }
+        }
+    }
+
+    // break on multi-values
+    public void testSkipperAllEqualValueWithMultiValues() throws IOException {
+        final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+        config.setIndexSort(new Sort(new SortField("sort", SortField.Type.LONG, false)));
+        try (Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+            final int gaps = random().nextInt(2, 10);
+            final int[] numDocs = new int[gaps];
+            long totaldocs = 0;
+            for (int i = 0; i < gaps; i++) {
+                int docs = random().nextInt(10) + 16;
+                numDocs[i] += docs;
+                for (int j = 0; j < docs; j++) {
+                    final Document doc = new Document();
+                    doc.add(new NumericDocValuesField("sort", totaldocs++));
+                    doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+                    writer.addDocument(doc);
+                }
+                if (i != gaps - 1) {
+                    // add doc with mutivalues
+                    final Document doc = new Document();
+                    doc.add(new NumericDocValuesField("sort", totaldocs++));
+                    doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+                    doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+                    writer.addDocument(doc);
+                    numDocs[i + 1] = 1;
+                }
+            }
+            writer.forceMerge(1);
+            try (IndexReader reader = writer.getReader()) {
+                assertEquals(1, reader.leaves().size());
+                final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+                assertNotNull(skipper);
+                assertEquals(Arrays.stream(numDocs).sum(), skipper.docCount());
+                skipper.advance(0);
+                for (int i = 0; i < gaps; i++) {
+                    assertEquals(0L, skipper.minValue(0));
+                    assertEquals(0L, skipper.maxValue(0));
+                    assertEquals(numDocs[i], skipper.docCount(0));
+                    skipper.advance(skipper.maxDocID(0) + 1);
+                }
+                assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/BaseKnnBitVectorsFormatTestCase.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/BaseKnnBitVectorsFormatTestCase.java
index 8f0a306e1eb3b..86b60d9984de5 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/BaseKnnBitVectorsFormatTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/BaseKnnBitVectorsFormatTestCase.java
@@ -19,6 +19,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
@@ -110,8 +111,9 @@ public void testRandom() throws Exception {
                     totalSize += vectorValues.size();
                     StoredFields storedFields = ctx.reader().storedFields();
                     int docId;
-                    while ((docId = vectorValues.nextDoc()) != NO_MORE_DOCS) {
-                        byte[] v = vectorValues.vectorValue();
+                    KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
+                    while ((docId = iterator.nextDoc()) != NO_MORE_DOCS) {
+                        byte[] v = vectorValues.vectorValue(iterator.index());
                         assertEquals(dimension, v.length);
                         String idString = storedFields.document(docId).getField("id").stringValue();
                         int id = Integer.parseInt(idString);
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java
index aa50bc26c4443..57cca6eea86ec 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
 import org.elasticsearch.common.logging.LogConfigurator;
 
@@ -24,7 +24,7 @@ public class ES813FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase {
 
     @Override
     protected Codec getCodec() {
-        return new Lucene912Codec() {
+        return new Lucene100Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES813FlatVectorFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java
index 8cb927036588a..9069b094ee483 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
 import org.elasticsearch.common.logging.LogConfigurator;
 
@@ -24,7 +24,7 @@ public class ES813Int8FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase
 
     @Override
     protected Codec getCodec() {
-        return new Lucene912Codec() {
+        return new Lucene100Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES813Int8FlatVectorFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java
index cee60efb57327..549a14ca6c31b 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.KnnFloatVectorField;
@@ -19,6 +19,7 @@
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.VectorSimilarityFunction;
@@ -41,7 +42,7 @@ public class ES814HnswScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFo
 
     @Override
     protected Codec getCodec() {
-        return new Lucene912Codec() {
+        return new Lucene100Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES814HnswScalarQuantizedVectorsFormat();
@@ -68,9 +69,10 @@ public void testAddIndexesDirectory0FS() throws Exception {
                 try (IndexReader reader = DirectoryReader.open(w2)) {
                     LeafReader r = getOnlyLeafReader(reader);
                     FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
-                    assertEquals(0, vectorValues.nextDoc());
-                    assertEquals(0, vectorValues.vectorValue()[0], 0);
-                    assertEquals(NO_MORE_DOCS, vectorValues.nextDoc());
+                    KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
+                    assertEquals(0, iterator.nextDoc());
+                    assertEquals(0, vectorValues.vectorValue(iterator.index())[0], 0);
+                    assertEquals(NO_MORE_DOCS, iterator.nextDoc());
                 }
             }
         }
@@ -110,12 +112,13 @@ private void testAddIndexesDirectory01FS(VectorSimilarityFunction similarityFunc
                 try (IndexReader reader = DirectoryReader.open(w2)) {
                     LeafReader r = getOnlyLeafReader(reader);
                     FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
-                    assertEquals(0, vectorValues.nextDoc());
+                    KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
+                    assertEquals(0, iterator.nextDoc());
                     // The merge order is randomized, we might get 1 first, or 2
-                    float value = vectorValues.vectorValue()[0];
+                    float value = vectorValues.vectorValue(iterator.index())[0];
                     assertTrue(value == 1 || value == 2);
-                    assertEquals(1, vectorValues.nextDoc());
-                    value += vectorValues.vectorValue()[0];
+                    assertEquals(1, iterator.nextDoc());
+                    value += vectorValues.vectorValue(iterator.index())[0];
                     assertEquals(3f, value, 0);
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java
index 90d2584feb3f2..034d428b25209 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.junit.Before;
 
@@ -19,7 +19,7 @@ public class ES815BitFlatVectorFormatTests extends BaseKnnBitVectorsFormatTestCa
 
     @Override
     protected Codec getCodec() {
-        return new Lucene912Codec() {
+        return new Lucene100Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES815BitFlatVectorFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java
index add90ea271fa1..4af6a405c7705 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.junit.Before;
 
@@ -19,7 +19,7 @@ public class ES815HnswBitVectorsFormatTests extends BaseKnnBitVectorsFormatTestC
 
     @Override
     protected Codec getCodec() {
-        return new Lucene912Codec() {
+        return new Lucene100Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES815HnswBitVectorsFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java
index 4ac66a9f63a3f..cef5e5358f3d5 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.codec.vectors;
 
 import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.VectorScorer;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.common.logging.LogConfigurator;
@@ -61,7 +62,7 @@ public void testScore() throws IOException {
             new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, normVmC, vDotC)
         );
 
-        RandomAccessBinarizedByteVectorValues targetVectors = new RandomAccessBinarizedByteVectorValues() {
+        BinarizedByteVectorValues targetVectors = new BinarizedByteVectorValues() {
             @Override
             public float getCentroidDistance(int vectorOrd) throws IOException {
                 return random().nextFloat(0f, 1000f);
@@ -99,7 +100,7 @@ public float[] getCentroid() throws IOException {
             }
 
             @Override
-            public RandomAccessBinarizedByteVectorValues copy() throws IOException {
+            public BinarizedByteVectorValues copy() throws IOException {
                 return null;
             }
 
@@ -115,6 +116,16 @@ public int size() {
                 return 1;
             }
 
+            @Override
+            public VectorScorer scorer(float[] query) throws IOException {
+                return null;
+            }
+
+            @Override
+            public float[] getCorrectiveTerms(int vectorOrd) throws IOException {
+                return new float[0];
+            }
+
             @Override
             public int dimension() {
                 return dimensions;
@@ -209,7 +220,7 @@ public void testScoreEuclidean() throws IOException {
             new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, 0f, 0f)
         );
 
-        RandomAccessBinarizedByteVectorValues targetVectors = new RandomAccessBinarizedByteVectorValues() {
+        BinarizedByteVectorValues targetVectors = new BinarizedByteVectorValues() {
             @Override
             public float getCentroidDistance(int vectorOrd) {
                 return 355.78073f;
@@ -375,7 +386,7 @@ public float[] getCentroid() {
             }
 
             @Override
-            public RandomAccessBinarizedByteVectorValues copy() {
+            public BinarizedByteVectorValues copy() {
                 return null;
             }
 
@@ -389,6 +400,16 @@ public int size() {
                 return 1;
             }
 
+            @Override
+            public VectorScorer scorer(float[] query) throws IOException {
+                return null;
+            }
+
+            @Override
+            public float[] getCorrectiveTerms(int vectorOrd) throws IOException {
+                return new float[0];
+            }
+
             @Override
             public int dimension() {
                 return dimensions;
@@ -806,7 +827,7 @@ public void testScoreMIP() throws IOException {
             new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, normVmC, vDotC)
         );
 
-        RandomAccessBinarizedByteVectorValues targetVectors = new RandomAccessBinarizedByteVectorValues() {
+        BinarizedByteVectorValues targetVectors = new BinarizedByteVectorValues() {
             @Override
             public float getCentroidDistance(int vectorOrd) {
                 return 0f;
@@ -1617,7 +1638,7 @@ public float[] getCentroid() {
             }
 
             @Override
-            public RandomAccessBinarizedByteVectorValues copy() {
+            public BinarizedByteVectorValues copy() {
                 return null;
             }
 
@@ -1727,6 +1748,16 @@ public int size() {
                 return 1;
             }
 
+            @Override
+            public VectorScorer scorer(float[] query) throws IOException {
+                return null;
+            }
+
+            @Override
+            public float[] getCorrectiveTerms(int vectorOrd) throws IOException {
+                return new float[0];
+            }
+
             @Override
             public int dimension() {
                 return dimensions;
@@ -1741,6 +1772,6 @@ public int dimension() {
             similarityFunction
         );
 
-        assertEquals(132.30249f, scorer.score(0), 0.0001f);
+        assertEquals(129.64046f, scorer.score(0), 0.0001f);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java
index 0892436891ff1..42f2fbb383ac9 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java
@@ -22,7 +22,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.index.DirectoryReader;
@@ -30,6 +30,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.search.IndexSearcher;
@@ -58,7 +59,7 @@ public class ES816BinaryQuantizedVectorsFormatTests extends BaseKnnVectorsFormat
 
     @Override
     protected Codec getCodec() {
-        return new Lucene912Codec() {
+        return new Lucene100Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES816BinaryQuantizedVectorsFormat();
@@ -90,8 +91,8 @@ public void testSearch() throws Exception {
                     float[] queryVector = randomVector(dims);
                     Query q = new KnnFloatVectorQuery(fieldName, queryVector, k);
                     TopDocs collectedDocs = searcher.search(q, k);
-                    assertEquals(k, collectedDocs.totalHits.value);
-                    assertEquals(TotalHits.Relation.EQUAL_TO, collectedDocs.totalHits.relation);
+                    assertEquals(k, collectedDocs.totalHits.value());
+                    assertEquals(TotalHits.Relation.EQUAL_TO, collectedDocs.totalHits.relation());
                 }
             }
         }
@@ -148,7 +149,7 @@ public void testQuantizedVectorsWriteAndRead() throws IOException {
                     LeafReader r = getOnlyLeafReader(reader);
                     FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
                     assertEquals(vectorValues.size(), numVectors);
-                    OffHeapBinarizedVectorValues qvectorValues = ((ES816BinaryQuantizedVectorsReader.BinarizedVectorValues) vectorValues)
+                    BinarizedByteVectorValues qvectorValues = ((ES816BinaryQuantizedVectorsReader.BinarizedVectorValues) vectorValues)
                         .getQuantizedVectorValues();
                     float[] centroid = qvectorValues.getCentroid();
                     assertEquals(centroid.length, dims);
@@ -159,13 +160,18 @@ public void testQuantizedVectorsWriteAndRead() throws IOException {
                     if (similarityFunction == VectorSimilarityFunction.COSINE) {
                         vectorValues = new ES816BinaryQuantizedVectorsWriter.NormalizedFloatVectorValues(vectorValues);
                     }
-
-                    while (vectorValues.nextDoc() != NO_MORE_DOCS) {
-                        float[] corrections = quantizer.quantizeForIndex(vectorValues.vectorValue(), expectedVector, centroid);
-                        assertArrayEquals(expectedVector, qvectorValues.vectorValue());
-                        assertEquals(corrections.length, qvectorValues.getCorrectiveTerms().length);
+                    KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator();
+
+                    while (docIndexIterator.nextDoc() != NO_MORE_DOCS) {
+                        float[] corrections = quantizer.quantizeForIndex(
+                            vectorValues.vectorValue(docIndexIterator.index()),
+                            expectedVector,
+                            centroid
+                        );
+                        assertArrayEquals(expectedVector, qvectorValues.vectorValue(docIndexIterator.index()));
+                        assertEquals(corrections.length, qvectorValues.getCorrectiveTerms(docIndexIterator.index()).length);
                         for (int i = 0; i < corrections.length; i++) {
-                            assertEquals(corrections[i], qvectorValues.getCorrectiveTerms()[i], 0.00001f);
+                            assertEquals(corrections[i], qvectorValues.getCorrectiveTerms(docIndexIterator.index())[i], 0.00001f);
                         }
                     }
                 }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java
index f607de57e1fd5..ca96e093b7b28 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java
@@ -22,7 +22,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.KnnFloatVectorField;
@@ -30,6 +30,7 @@
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.search.TopDocs;
@@ -55,7 +56,7 @@ public class ES816HnswBinaryQuantizedVectorsFormatTests extends BaseKnnVectorsFo
 
     @Override
     protected Codec getCodec() {
-        return new Lucene912Codec() {
+        return new Lucene100Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES816HnswBinaryQuantizedVectorsFormat();
@@ -91,12 +92,13 @@ public void testSingleVectorCase() throws Exception {
                 try (IndexReader reader = DirectoryReader.open(w)) {
                     LeafReader r = getOnlyLeafReader(reader);
                     FloatVectorValues vectorValues = r.getFloatVectorValues("f");
+                    KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator();
                     assert (vectorValues.size() == 1);
-                    while (vectorValues.nextDoc() != NO_MORE_DOCS) {
-                        assertArrayEquals(vector, vectorValues.vectorValue(), 0.00001f);
+                    while (docIndexIterator.nextDoc() != NO_MORE_DOCS) {
+                        assertArrayEquals(vector, vectorValues.vectorValue(docIndexIterator.index()), 0.00001f);
                     }
                     TopDocs td = r.searchNearestVectors("f", randomVector(vector.length), 1, null, Integer.MAX_VALUE);
-                    assertEquals(1, td.totalHits.value);
+                    assertEquals(1, td.totalHits.value());
                     assertTrue(td.scoreDocs[0].score >= 0);
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
index c3fea6c7a189b..437ba1cecc11d 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
@@ -10,7 +10,7 @@
 package org.elasticsearch.index.codec.zstd;
 
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.DirectoryReader;
@@ -35,13 +35,13 @@ public class StoredFieldCodecDuelTests extends ESTestCase {
     private static final String DOUBLE_FIELD = "double_field_5";
 
     public void testDuelBestSpeed() throws IOException {
-        var baseline = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
+        var baseline = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
         var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
         doTestDuel(baseline, contender);
     }
 
     public void testDuelBestCompression() throws IOException {
-        var baseline = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
+        var baseline = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
         var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
         doTestDuel(baseline, contender);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java
index 71c7464657e72..77a7585e3b518 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java
@@ -11,11 +11,11 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.elasticsearch.index.codec.Elasticsearch816Codec;
+import org.elasticsearch.index.codec.Elasticsearch900Codec;
 
 public class Zstd814BestCompressionStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {
 
-    private final Codec codec = new Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION);
+    private final Codec codec = new Elasticsearch900Codec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION);
 
     @Override
     protected Codec getCodec() {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java
index 02a1b10697907..3d6cfea70d121 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java
@@ -11,11 +11,11 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.elasticsearch.index.codec.Elasticsearch816Codec;
+import org.elasticsearch.index.codec.Elasticsearch900Codec;
 
 public class Zstd814BestSpeedStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {
 
-    private final Codec codec = new Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
+    private final Codec codec = new Elasticsearch900Codec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
 
     @Override
     protected Codec getCodec() {
diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
index 6565a11a860ec..6d205a22433b4 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene100.Lucene100Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -45,7 +45,7 @@ public void testExceptionsAreNotCached() {
     public void testCompletionStatsCache() throws IOException, InterruptedException {
         final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
         final PostingsFormat postingsFormat = new Completion912PostingsFormat();
-        indexWriterConfig.setCodec(new Lucene912Codec() {
+        indexWriterConfig.setCodec(new Lucene100Codec() {
             @Override
             public PostingsFormat getPostingsFormatForField(String field) {
                 return postingsFormat; // all fields are suggest fields
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index c8ca3d17de797..21aefd893de70 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -266,7 +266,7 @@ public void testVersionMapAfterAutoIDDocument() throws IOException {
         try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
             assertEquals(1, searcher.getIndexReader().numDocs());
             TopDocs search = searcher.search(new MatchAllDocsQuery(), 1);
-            org.apache.lucene.document.Document luceneDoc = searcher.doc(search.scoreDocs[0].doc);
+            org.apache.lucene.document.Document luceneDoc = searcher.storedFields().document(search.scoreDocs[0].doc);
             assertEquals("test", luceneDoc.get("value"));
         }
 
@@ -279,7 +279,7 @@ public void testVersionMapAfterAutoIDDocument() throws IOException {
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             assertEquals(1, searcher.getIndexReader().numDocs());
             TopDocs search = searcher.search(new MatchAllDocsQuery(), 1);
-            org.apache.lucene.document.Document luceneDoc = searcher.doc(search.scoreDocs[0].doc);
+            org.apache.lucene.document.Document luceneDoc = searcher.storedFields().document(search.scoreDocs[0].doc);
             assertEquals("updated", luceneDoc.get("value"));
         }
 
@@ -640,7 +640,7 @@ public void testTranslogMultipleOperationsSameDocument() throws IOException {
             recoverFromTranslog(recoveringEngine, translogHandler, Long.MAX_VALUE);
             recoveringEngine.refresh("test");
             try (Engine.Searcher searcher = recoveringEngine.acquireSearcher("test")) {
-                Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+                Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()));
                 assertThat(totalHits, equalTo(operations.get(operations.size() - 1) instanceof Engine.Delete ? 0 : 1));
             }
         }
@@ -747,7 +747,7 @@ public void testTranslogRecoveryWithMultipleGenerations() throws IOException {
             recoveringEngine.refresh("test");
             try (Engine.Searcher searcher = recoveringEngine.acquireSearcher("test")) {
                 TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), docs);
-                assertEquals(docs, topDocs.totalHits.value);
+                assertEquals(docs, topDocs.totalHits.value());
             }
         } finally {
             IOUtils.close(initialEngine, recoveringEngine, store);
@@ -2010,7 +2010,7 @@ public void testConcurrentOutOfOrderDocsOnReplica() throws IOException, Interrup
             try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                 Integer totalHits = searcher.search(
                     new TermQuery(new Term("value", lastFieldValueDoc1)),
-                    new TotalHitCountCollectorManager()
+                    new TotalHitCountCollectorManager(searcher.getSlices())
                 );
                 assertThat(totalHits, equalTo(1));
             }
@@ -2019,7 +2019,7 @@ public void testConcurrentOutOfOrderDocsOnReplica() throws IOException, Interrup
             try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                 Integer totalHits = searcher.search(
                     new TermQuery(new Term("value", lastFieldValueDoc2)),
-                    new TotalHitCountCollectorManager()
+                    new TotalHitCountCollectorManager(searcher.getSlices())
                 );
                 assertThat(totalHits, equalTo(1));
             }
@@ -2249,7 +2249,7 @@ private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion
                     try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                         Integer totalHits = searcher.search(
                             new TermQuery(new Term("value", lastFieldValue)),
-                            new TotalHitCountCollectorManager()
+                            new TotalHitCountCollectorManager(searcher.getSlices())
                         );
                         assertThat(totalHits, equalTo(1));
                     }
@@ -2275,7 +2275,10 @@ private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion
         assertVisibleCount(engine, docDeleted ? 0 : 1);
         if (docDeleted == false) {
             try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-                Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager());
+                Integer totalHits = searcher.search(
+                    new TermQuery(new Term("value", lastFieldValue)),
+                    new TotalHitCountCollectorManager(searcher.getSlices())
+                );
                 assertThat(totalHits, equalTo(1));
             }
         }
@@ -2361,7 +2364,10 @@ public void testNonInternalVersioningOnPrimary() throws IOException {
         if (docDeleted == false) {
             logger.info("searching for [{}]", lastFieldValue);
             try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-                Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager());
+                Integer totalHits = searcher.search(
+                    new TermQuery(new Term("value", lastFieldValue)),
+                    new TotalHitCountCollectorManager(searcher.getSlices())
+                );
                 assertThat(totalHits, equalTo(1));
             }
         }
@@ -2378,7 +2384,7 @@ public void testVersioningPromotedReplica() throws IOException {
         final int opsOnPrimary = assertOpsOnPrimary(primaryOps, finalReplicaVersion, deletedOnReplica, replicaEngine);
         final long currentSeqNo = getSequenceID(replicaEngine, new Engine.Get(false, false, Term.toString(lastReplicaOp.uid()))).v1();
         try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
-            Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+            Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()));
             if (totalHits > 0) {
                 // last op wasn't delete
                 assertThat(currentSeqNo, equalTo(finalReplicaSeqNo + opsOnPrimary));
@@ -2402,7 +2408,10 @@ public void testConcurrentExternalVersioningOnPrimary() throws IOException, Inte
         assertVisibleCount(engine, lastFieldValue == null ? 0 : 1);
         if (lastFieldValue != null) {
             try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-                Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager());
+                Integer totalHits = searcher.search(
+                    new TermQuery(new Term("value", lastFieldValue)),
+                    new TotalHitCountCollectorManager(searcher.getSlices())
+                );
                 assertThat(totalHits, equalTo(1));
             }
         }
@@ -2434,7 +2443,7 @@ class OpAndVersion {
                 Engine.Get engineGet = new Engine.Get(true, false, doc.id());
                 try (Engine.GetResult get = engine.get(engineGet, mappingLookup, documentParser, randomSearcherWrapper())) {
                     FieldsVisitor visitor = new FieldsVisitor(true);
-                    get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
+                    get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor);
                     List<String> values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString()));
                     String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null;
                     String added = "v_" + idGenerator.incrementAndGet();
@@ -2480,7 +2489,7 @@ class OpAndVersion {
             Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), mappingLookup, documentParser, randomSearcherWrapper())
         ) {
             FieldsVisitor visitor = new FieldsVisitor(true);
-            get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
+            get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor);
             List<String> values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString()));
             assertThat(currentValues, equalTo(new HashSet<>(values)));
         }
@@ -3424,7 +3433,7 @@ public void testSkipTranslogReplay() throws IOException {
             engine.skipTranslogRecovery();
             try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
                 TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
-                assertThat(topDocs.totalHits.value, equalTo(0L));
+                assertThat(topDocs.totalHits.value(), equalTo(0L));
             }
         }
     }
@@ -3514,7 +3523,7 @@ public void testTranslogReplay() throws IOException {
         assertThat(result.getVersion(), equalTo(2L));
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), numDocs + 1);
-            assertThat(topDocs.totalHits.value, equalTo(numDocs + 1L));
+            assertThat(topDocs.totalHits.value(), equalTo(numDocs + 1L));
         }
 
         engine.close();
@@ -3523,7 +3532,7 @@ public void testTranslogReplay() throws IOException {
         engine.refresh("warm_up");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), numDocs + 1);
-            assertThat(topDocs.totalHits.value, equalTo(numDocs + 1L));
+            assertThat(topDocs.totalHits.value(), equalTo(numDocs + 1L));
         }
         assertEquals(flush ? 1 : 2, translogHandler.appliedOperations());
         engine.delete(new Engine.Delete(Integer.toString(randomId), newUid(doc), primaryTerm.get()));
@@ -3534,7 +3543,7 @@ public void testTranslogReplay() throws IOException {
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), numDocs);
-            assertThat(topDocs.totalHits.value, equalTo((long) numDocs));
+            assertThat(topDocs.totalHits.value(), equalTo((long) numDocs));
         }
     }
 
@@ -3890,7 +3899,7 @@ public void testDoubleDeliveryPrimary() throws IOException {
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
         operation = appendOnlyPrimary(doc, false, 1, create);
         retry = appendOnlyPrimary(doc, true, 1, create);
@@ -3925,7 +3934,7 @@ public void testDoubleDeliveryPrimary() throws IOException {
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
     }
 
@@ -3983,7 +3992,7 @@ public void testDoubleDeliveryReplicaAppendingAndDeleteOnly() throws IOException
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(0, topDocs.totalHits.value);
+            assertEquals(0, topDocs.totalHits.value());
         }
     }
 
@@ -4007,7 +4016,7 @@ public void testDoubleDeliveryReplicaAppendingOnly() throws IOException {
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
 
         final boolean create = randomBoolean();
@@ -4047,7 +4056,7 @@ public void testDoubleDeliveryReplicaAppendingOnly() throws IOException {
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
     }
 
@@ -4092,12 +4101,12 @@ public void testDoubleDeliveryReplica() throws IOException {
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
         if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
             List<Translog.Operation> ops = readAllOperationsInLucene(engine);
@@ -4172,7 +4181,7 @@ public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOExcep
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
 
         index = new Engine.Index(
@@ -4194,7 +4203,7 @@ public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOExcep
         replicaEngine.refresh("test");
         try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
     }
 
@@ -4264,7 +4273,7 @@ public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs()
         engine.refresh("test");
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
 
         Engine.Index secondIndexRequestReplica = new Engine.Index(
@@ -4285,7 +4294,7 @@ public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs()
         replicaEngine.refresh("test");
         try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10);
-            assertEquals(1, topDocs.totalHits.value);
+            assertEquals(1, topDocs.totalHits.value());
         }
     }
 
@@ -5678,7 +5687,7 @@ public void testConcurrentAppendUpdateAndRefresh() throws InterruptedException,
         try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
             TopDocs search = searcher.search(new MatchAllDocsQuery(), searcher.getIndexReader().numDocs());
             for (int i = 0; i < search.scoreDocs.length; i++) {
-                org.apache.lucene.document.Document luceneDoc = searcher.doc(search.scoreDocs[i].doc);
+                org.apache.lucene.document.Document luceneDoc = searcher.storedFields().document(search.scoreDocs[i].doc);
                 assertEquals("updated", luceneDoc.get("value"));
             }
             int totalNumDocs = numDocs - numDeletes.get();
@@ -6666,7 +6675,7 @@ public void testStoreHonorsLuceneVersion() throws IOException {
                 engine.refresh("test");
                 try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                     LeafReader leafReader = getOnlyLeafReader(searcher.getIndexReader());
-                    assertEquals(createdVersion.luceneVersion().major, leafReader.getMetaData().getCreatedVersionMajor());
+                    assertEquals(createdVersion.luceneVersion().major, leafReader.getMetaData().createdVersionMajor());
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
index a3a21fc32e546..b6be13b9f2513 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Tuple;
@@ -71,22 +70,16 @@ public void testRamBytesUsed() throws Exception {
         }
         actualRamBytesUsed = RamUsageTester.ramUsed(map);
         estimatedRamBytesUsed = map.ramBytesUsed();
-        long tolerance;
-        if (Constants.JRE_IS_MINIMUM_JAVA9) {
-            // With Java 9, RamUsageTester computes the memory usage of maps as
-            // the memory usage of an array that would contain exactly all keys
-            // and values. This is an under-estimation of the actual memory
-            // usage since it ignores the impact of the load factor and of the
-            // linked list/tree that is used to resolve collisions. So we use a
-            // bigger tolerance.
-            // less than 50% off
-            tolerance = actualRamBytesUsed / 2;
-        } else {
-            // Java 8 is more accurate by doing reflection into the actual JDK classes
-            // so we give it a lower error bound.
-            // less than 25% off
-            tolerance = actualRamBytesUsed / 4;
-        }
+
+        // Since Java 9, RamUsageTester computes the memory usage of maps as
+        // the memory usage of an array that would contain exactly all keys
+        // and values. This is an under-estimation of the actual memory
+        // usage since it ignores the impact of the load factor and of the
+        // linked list/tree that is used to resolve collisions. So we use a
+        // bigger tolerance.
+        // less than 50% off
+        long tolerance = actualRamBytesUsed / 2;
+
         assertEquals(actualRamBytesUsed, estimatedRamBytesUsed, tolerance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java
index e7e668415cdd4..c0e365909429a 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.StandardDirectoryReader;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -69,8 +70,9 @@ public void testPruneAll() throws IOException {
                 writer.forceMerge(1);
                 writer.commit();
                 try (DirectoryReader reader = DirectoryReader.open(writer)) {
+                    StoredFields storedFields = reader.storedFields();
                     for (int i = 0; i < reader.maxDoc(); i++) {
-                        Document document = reader.document(i);
+                        Document document = storedFields.document(i);
                         if (pruneIdField) {
                             assertEquals(1, document.getFields().size());
                             assertEquals("source", document.getFields().get(0).name());
@@ -151,8 +153,9 @@ public void testPruneSome() throws IOException {
                     assertEquals(1, reader.leaves().size());
                     NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source");
                     assertNotNull(extra_source);
+                    StoredFields storedFields = reader.storedFields();
                     for (int i = 0; i < reader.maxDoc(); i++) {
-                        Document document = reader.document(i);
+                        Document document = storedFields.document(i);
                         Set<String> collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet());
                         assertTrue(collect.contains("source"));
                         assertTrue(collect.contains("even"));
@@ -192,8 +195,9 @@ public void testPruneNone() throws IOException {
                     assertEquals(1, reader.leaves().size());
                     NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source");
                     assertNotNull(extra_source);
+                    StoredFields storedFields = reader.storedFields();
                     for (int i = 0; i < reader.maxDoc(); i++) {
-                        Document document = reader.document(i);
+                        Document document = storedFields.document(i);
                         Set<String> collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet());
                         assertTrue(collect.contains("source"));
                         assertTrue(collect.contains("extra_source"));
diff --git a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java
index a6e56c4137028..49036324e722e 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/SegmentTests.java
@@ -80,7 +80,7 @@ static Segment randomSegment() {
         segment.sizeInBytes = randomNonNegativeLong();
         segment.docCount = randomIntBetween(1, Integer.MAX_VALUE);
         segment.delDocCount = randomIntBetween(0, segment.docCount);
-        segment.version = Version.LUCENE_8_0_0;
+        segment.version = Version.LUCENE_9_0_0;
         segment.compound = randomBoolean();
         segment.mergeId = randomAlphaOfLengthBetween(1, 10);
         segment.segmentSort = randomIndexSort();
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
index b7793a644f8b8..9d0a9cdeb1968 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
@@ -113,7 +113,7 @@ public void testSingleValueAllSet() throws Exception {
             TopFieldDocs topDocs;
             SortField sortField = indexFieldData.sortField(null, MultiValueMode.MIN, null, false);
             topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
-            assertThat(topDocs.totalHits.value, equalTo(3L));
+            assertThat(topDocs.totalHits.value(), equalTo(3L));
             assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
             assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
             assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@@ -123,7 +123,7 @@ public void testSingleValueAllSet() throws Exception {
 
             sortField = indexFieldData.sortField(null, MultiValueMode.MAX, null, true);
             topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
-            assertThat(topDocs.totalHits.value, equalTo(3L));
+            assertThat(topDocs.totalHits.value(), equalTo(3L));
             assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
             assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
             assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
@@ -193,14 +193,14 @@ public void testMultiValueAllSet() throws Exception {
             IndexSearcher searcher = newIndexSearcher(DirectoryReader.open(writer));
             SortField sortField = indexFieldData.sortField(null, MultiValueMode.MIN, null, false);
             TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
-            assertThat(topDocs.totalHits.value, equalTo(3L));
+            assertThat(topDocs.totalHits.value(), equalTo(3L));
             assertThat(topDocs.scoreDocs.length, equalTo(3));
             assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
             assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
             assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
             sortField = indexFieldData.sortField(null, MultiValueMode.MAX, null, true);
             topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
-            assertThat(topDocs.totalHits.value, equalTo(3L));
+            assertThat(topDocs.totalHits.value(), equalTo(3L));
             assertThat(topDocs.scoreDocs.length, equalTo(3));
             assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
             assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
@@ -258,7 +258,7 @@ public void testSortMultiValuesFields() throws Exception {
         IndexSearcher searcher = newIndexSearcher(DirectoryReader.open(writer));
         SortField sortField = indexFieldData.sortField(null, MultiValueMode.MIN, null, false);
         TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
-        assertThat(topDocs.totalHits.value, equalTo(8L));
+        assertThat(topDocs.totalHits.value(), equalTo(8L));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("!08"));
@@ -279,7 +279,7 @@ public void testSortMultiValuesFields() throws Exception {
 
         sortField = indexFieldData.sortField(null, MultiValueMode.MAX, null, true);
         topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
-        assertThat(topDocs.totalHits.value, equalTo(8L));
+        assertThat(topDocs.totalHits.value(), equalTo(8L));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("10"));
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
index 83f668f20de7b..f809a53d753fb 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.shard.ShardId;
@@ -94,7 +95,7 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 fieldType = new TextFieldMapper.Builder(
                     fieldName,
                     createDefaultIndexAnalyzers(),
-                    indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+                    SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
                 ).fielddata(true).build(context).fieldType();
             }
         } else if (type.equals("float")) {
@@ -162,10 +163,9 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 docValues
             ).build(context).fieldType();
         } else if (type.equals("binary")) {
-            fieldType = new BinaryFieldMapper.Builder(fieldName, indexService.getIndexSettings().getMode().isSyntheticSourceEnabled())
-                .docValues(docValues)
-                .build(context)
-                .fieldType();
+            fieldType = new BinaryFieldMapper.Builder(fieldName, SourceFieldMapper.isSynthetic(indexService.getIndexSettings())).docValues(
+                docValues
+            ).build(context).fieldType();
         } else {
             throw new UnsupportedOperationException(type);
         }
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
index ac77f147a7ce6..48d6cabefe345 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
@@ -262,10 +262,10 @@ public void testActualMissingValue(boolean reverse) throws IOException {
             randomBoolean() ? numDocs : randomIntBetween(10, numDocs),
             new Sort(sortField)
         );
-        assertEquals(numDocs, topDocs.totalHits.value);
+        assertEquals(numDocs, topDocs.totalHits.value());
         BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
         for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
-            final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value");
+            final String docValue = searcher.storedFields().document(topDocs.scoreDocs[i].doc).get("value");
             final BytesRef value = new BytesRef(docValue == null ? missingValue : docValue);
             if (reverse) {
                 assertTrue(previousValue.compareTo(value) >= 0);
@@ -321,10 +321,10 @@ public void testSortMissing(boolean first, boolean reverse) throws IOException {
             new Sort(sortField)
         );
 
-        assertThat(topDocs.totalHits.value, lessThanOrEqualTo((long) numDocs));
+        assertThat(topDocs.totalHits.value(), lessThanOrEqualTo((long) numDocs));
         BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
         for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
-            final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value");
+            final String docValue = searcher.storedFields().document(topDocs.scoreDocs[i].doc).get("value");
             if (first && docValue == null) {
                 assertNull(previousValue);
             } else if (first == false && docValue != null) {
@@ -414,7 +414,7 @@ public void testNestedSorting(MultiValueMode sortMode) throws IOException {
             assertTrue("expected " + docID + " to be a parent", parents.get(docID));
             BytesRef cmpValue = null;
             for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) {
-                String[] sVals = searcher.doc(child).getValues("text");
+                String[] sVals = searcher.storedFields().document(child).getValues("text");
                 final BytesRef[] vals;
                 if (sVals.length == 0) {
                     vals = new BytesRef[0];
@@ -498,15 +498,11 @@ public void testGlobalOrdinals() throws Exception {
         ord = values.nextOrd();
         assertThat(ord, equalTo(5L));
         assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("04"));
-        ord = values.nextOrd();
-        assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS));
         assertFalse(values.advanceExact(1));
         assertTrue(values.advanceExact(2));
         ord = values.nextOrd();
         assertThat(ord, equalTo(4L));
         assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("03"));
-        ord = values.nextOrd();
-        assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS));
 
         // Second segment
         leaf = topLevelReader.leaves().get(1);
@@ -522,8 +518,6 @@ public void testGlobalOrdinals() throws Exception {
         ord = values.nextOrd();
         assertThat(ord, equalTo(7L));
         assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("06"));
-        ord = values.nextOrd();
-        assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS));
         assertTrue(values.advanceExact(1));
         ord = values.nextOrd();
         assertThat(ord, equalTo(7L));
@@ -534,8 +528,6 @@ public void testGlobalOrdinals() throws Exception {
         ord = values.nextOrd();
         assertThat(ord, equalTo(9L));
         assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("08"));
-        ord = values.nextOrd();
-        assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS));
         assertFalse(values.advanceExact(2));
         assertTrue(values.advanceExact(3));
         ord = values.nextOrd();
@@ -547,8 +539,6 @@ public void testGlobalOrdinals() throws Exception {
         ord = values.nextOrd();
         assertThat(ord, equalTo(11L));
         assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("10"));
-        ord = values.nextOrd();
-        assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS));
 
         // Third segment
         leaf = topLevelReader.leaves().get(2);
@@ -564,8 +554,6 @@ public void testGlobalOrdinals() throws Exception {
         ord = values.nextOrd();
         assertThat(ord, equalTo(2L));
         assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("!10"));
-        ord = values.nextOrd();
-        assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS));
     }
 
     public void testTermsEnum() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
index b0a30211c0f47..a7277b79e5c00 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.index.SortedSetDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 
 import java.util.List;
@@ -56,7 +57,7 @@ public void testFilterByFrequency() throws Exception {
             MappedFieldType ft = new TextFieldMapper.Builder(
                 "high_freq",
                 createDefaultIndexAnalyzers(),
-                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+                SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
             ).fielddata(true).fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0).build(builderContext).fieldType();
             IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH);
             for (LeafReaderContext context : contexts) {
@@ -72,7 +73,7 @@ public void testFilterByFrequency() throws Exception {
             MappedFieldType ft = new TextFieldMapper.Builder(
                 "high_freq",
                 createDefaultIndexAnalyzers(),
-                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+                SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
             ).fielddata(true)
                 .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100)
                 .build(builderContext)
@@ -91,7 +92,7 @@ public void testFilterByFrequency() throws Exception {
             MappedFieldType ft = new TextFieldMapper.Builder(
                 "med_freq",
                 createDefaultIndexAnalyzers(),
-                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+                SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
             ).fielddata(true)
                 .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101)
                 .build(builderContext)
@@ -111,7 +112,7 @@ public void testFilterByFrequency() throws Exception {
             MappedFieldType ft = new TextFieldMapper.Builder(
                 "med_freq",
                 createDefaultIndexAnalyzers(),
-                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+                SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
             ).fielddata(true)
                 .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101)
                 .build(builderContext)
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
index 7616ea5119b6c..36c25b352a792 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
@@ -163,12 +164,12 @@ public void testClearField() throws Exception {
         final MappedFieldType mapper1 = new TextFieldMapper.Builder(
             "field_1",
             createDefaultIndexAnalyzers(),
-            indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+            SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
         ).fielddata(true).build(context).fieldType();
         final MappedFieldType mapper2 = new TextFieldMapper.Builder(
             "field_2",
             createDefaultIndexAnalyzers(),
-            indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+            SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
         ).fielddata(true).build(context).fieldType();
         final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
         Document doc = new Document();
@@ -234,7 +235,7 @@ public void testFieldDataCacheListener() throws Exception {
         final MappedFieldType mapper1 = new TextFieldMapper.Builder(
             "s",
             createDefaultIndexAnalyzers(),
-            indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+            SourceFieldMapper.isSynthetic(indexService.getIndexSettings())
         ).fielddata(true).build(context).fieldType();
         final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
         Document doc = new Document();
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java
index cb6732ce8bb7d..aa23dc6da19df 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java
@@ -108,7 +108,6 @@ public int compare(OrdAndId o1, OrdAndId o2) {
                     for (Long ord : docOrds) {
                         assertThat(docs.nextOrd(), equalTo(ord));
                     }
-                    assertEquals(SortedSetDocValues.NO_MORE_ORDS, docs.nextOrd());
                 }
                 for (int i = docId + 1; i < ordAndId.id; i++) {
                     assertFalse(singleOrds.advanceExact(i));
@@ -257,7 +256,6 @@ private void assertEquals(SortedSetDocValues docs, long[][] ordinalPlan) throws
                 for (long ord : ords) {
                     assertThat(docs.nextOrd(), equalTo(ord));
                 }
-                assertThat(docs.nextOrd(), equalTo(SortedSetDocValues.NO_MORE_ORDS));
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java b/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java
index 86ddbbbc97599..81be71aec23c8 100644
--- a/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java
@@ -47,7 +47,7 @@ public void testQueryRewriteOnRefresh() throws Exception {
                 .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("g")),
-            r1 -> assertThat(r1.getHits().getTotalHits().value, equalTo(3L))
+            r1 -> assertThat(r1.getHits().getTotalHits().value(), equalTo(3L))
         );
         assertRequestCacheStats(0, 1);
 
@@ -57,7 +57,7 @@ public void testQueryRewriteOnRefresh() throws Exception {
                 .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("g")),
-            r2 -> assertThat(r2.getHits().getTotalHits().value, equalTo(3L))
+            r2 -> assertThat(r2.getHits().getTotalHits().value(), equalTo(3L))
         );
         assertRequestCacheStats(1, 1);
 
@@ -72,7 +72,7 @@ public void testQueryRewriteOnRefresh() throws Exception {
                 .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setSize(0)
                 .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("g")),
-            r3 -> assertThat(r3.getHits().getTotalHits().value, equalTo(5L))
+            r3 -> assertThat(r3.getHits().getTotalHits().value(), equalTo(5L))
         );
         assertRequestCacheStats(1, 2);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
index 38552d0a58683..e0d94632f06c1 100644
--- a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
+++ b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
@@ -26,10 +25,8 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.RandomObjects;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -114,56 +111,6 @@ public void testToXContent() throws IOException {
         }
     }
 
-    public void testToCompatibleXContent() throws IOException {
-        {
-            GetResult getResult = new GetResult(
-                "index",
-                "id",
-                0,
-                1,
-                1,
-                true,
-                new BytesArray("""
-                    { "field1" : "value1", "field2":"value2"}"""),
-                singletonMap("field1", new DocumentField("field1", singletonList("value1"))),
-                singletonMap("field1", new DocumentField("metafield", singletonList("metavalue")))
-            );
-
-            try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) {
-                getResult.toXContent(builder, ToXContent.EMPTY_PARAMS);
-                String output = Strings.toString(builder);
-                assertEquals(XContentHelper.stripWhitespace("""
-                    {
-                      "_index": "index",
-                      "_type": "_doc",
-                      "_id": "id",
-                      "_version": 1,
-                      "_seq_no": 0,
-                      "_primary_term": 1,
-                      "metafield": "metavalue",
-                      "found": true,
-                      "_source": {
-                        "field1": "value1",
-                        "field2": "value2"
-                      },
-                      "fields": {
-                        "field1": [ "value1" ]
-                      }
-                    }"""), XContentHelper.stripWhitespace(output));
-            }
-        }
-        {
-            GetResult getResult = new GetResult("index", "id", UNASSIGNED_SEQ_NO, 0, 1, false, null, null, null);
-
-            try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) {
-                getResult.toXContent(builder, ToXContent.EMPTY_PARAMS);
-                String output = Strings.toString(builder);
-                assertEquals("""
-                    {"_index":"index","_type":"_doc","_id":"id","found":false}""", output);
-            }
-        }
-    }
-
     public void testToAndFromXContentEmbedded() throws Exception {
         XContentType xContentType = randomFrom(XContentType.values());
         Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
index ddba993fd41cc..4aa983a78b07b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
@@ -121,8 +122,15 @@ public void testSort() throws IOException {
                 BooleanScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null);
                 SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false);
                 TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf));
-                assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [false]}"));
-                assertThat(reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [true]}"));
+                StoredFields storedFields = reader.storedFields();
+                assertThat(
+                    storedFields.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [false]}")
+                );
+                assertThat(
+                    storedFields.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [true]}")
+                );
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
index 2c78f5f7fee20..f55d213bea581 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
@@ -38,25 +38,25 @@ public void testDoubleIndexingSameDoc() throws Exception {
         }, reader -> {
             IndexSearcher searcher = newSearcher(reader);
             TopDocs topDocs = searcher.search(mapperService.fieldType("field1").termQuery("value1", context), 10);
-            assertThat(topDocs.totalHits.value, equalTo(2L));
+            assertThat(topDocs.totalHits.value(), equalTo(2L));
 
             topDocs = searcher.search(mapperService.fieldType("field2").termQuery("1", context), 10);
-            assertThat(topDocs.totalHits.value, equalTo(2L));
+            assertThat(topDocs.totalHits.value(), equalTo(2L));
 
             topDocs = searcher.search(mapperService.fieldType("field3").termQuery("1.1", context), 10);
-            assertThat(topDocs.totalHits.value, equalTo(2L));
+            assertThat(topDocs.totalHits.value(), equalTo(2L));
 
             topDocs = searcher.search(mapperService.fieldType("field4").termQuery("2010-01-01", context), 10);
-            assertThat(topDocs.totalHits.value, equalTo(2L));
+            assertThat(topDocs.totalHits.value(), equalTo(2L));
 
             topDocs = searcher.search(mapperService.fieldType("field5").termQuery("1", context), 10);
-            assertThat(topDocs.totalHits.value, equalTo(2L));
+            assertThat(topDocs.totalHits.value(), equalTo(2L));
 
             topDocs = searcher.search(mapperService.fieldType("field5").termQuery("2", context), 10);
-            assertThat(topDocs.totalHits.value, equalTo(2L));
+            assertThat(topDocs.totalHits.value(), equalTo(2L));
 
             topDocs = searcher.search(mapperService.fieldType("field5").termQuery("3", context), 10);
-            assertThat(topDocs.totalHits.value, equalTo(2L));
+            assertThat(topDocs.totalHits.value(), equalTo(2L));
         });
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
index 48879cdd0d77e..140137015d98a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
@@ -120,9 +121,19 @@ public void testSort() throws IOException {
                 DoubleScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null);
                 SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false);
                 TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf));
-                assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1.1]}"));
-                assertThat(reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [2.1]}"));
-                assertThat(reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [4.2]}"));
+                StoredFields storedFields = reader.storedFields();
+                assertThat(
+                    storedFields.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [1.1]}")
+                );
+                assertThat(
+                    storedFields.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [2.1]}")
+                );
+                assertThat(
+                    storedFields.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [4.2]}")
+                );
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
index 358ad07664870..399740e6200e6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
@@ -69,7 +69,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti
         XContentParser parser = createParser(JsonXContent.jsonXContent, source);
         SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON);
 
-        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true).setSynthetic().build();
+        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false).setSynthetic().build();
         RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             new PassThroughObjectMapper.Builder("labels").setPriority(0).setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE)
         ).build(MapperBuilderContext.root(false, false));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
index 7f430cf676809..7e9a196faaa26 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
@@ -1377,6 +1377,7 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() {
     }
 
     public void testSubobjectsAutoFlatPaths() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createDynamicTemplateAutoSubobjects();
         ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
             b.field("foo.metric.count", 10);
@@ -1389,6 +1390,7 @@ public void testSubobjectsAutoFlatPaths() throws IOException {
     }
 
     public void testSubobjectsAutoStructuredPaths() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createDynamicTemplateAutoSubobjects();
         ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
             b.startObject("foo");
@@ -1411,6 +1413,7 @@ public void testSubobjectsAutoStructuredPaths() throws IOException {
     }
 
     public void testSubobjectsAutoArrayOfObjects() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createDynamicTemplateAutoSubobjects();
         ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
             b.startObject("foo");
@@ -1444,6 +1447,7 @@ public void testSubobjectsAutoArrayOfObjects() throws IOException {
     }
 
     public void testSubobjectAutoDynamicNested() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
             b.startArray("dynamic_templates");
             {
@@ -1482,6 +1486,7 @@ public void testSubobjectAutoDynamicNested() throws IOException {
     }
 
     public void testRootSubobjectAutoDynamicNested() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
             b.startArray("dynamic_templates");
             {
@@ -1515,6 +1520,7 @@ public void testRootSubobjectAutoDynamicNested() throws IOException {
     }
 
     public void testDynamicSubobjectsAutoDynamicFalse() throws Exception {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         // verify that we read the dynamic value properly from the parent mapper. DocumentParser#dynamicOrDefault splits the field
         // name where dots are found, but it does that only for the parent prefix e.g. metrics.service and not for the leaf suffix time.max
         DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
@@ -1578,6 +1584,7 @@ public void testDynamicSubobjectsAutoDynamicFalse() throws Exception {
     }
 
     public void testSubobjectsAutoWithInnerNestedFromDynamicTemplate() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
             b.startArray("dynamic_templates");
             {
@@ -2045,6 +2052,7 @@ public void testSubobjectsFalseFlattened() throws IOException {
     }
 
     public void testSubobjectsAutoFlattened() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         String mapping = """
             {
               "_doc": {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
index 4284bc00cfc15..0182da8ade48a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
@@ -78,5 +78,4 @@ public void testUsingEnabledSettingThrows() {
             ex.getMessage()
         );
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
index 72055940b8970..4cc447d97291c 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
@@ -19,6 +19,8 @@
 import org.elasticsearch.test.ESTestCase;
 import org.mockito.Mockito;
 
+import java.util.List;
+
 public class IdFieldTypeTests extends ESTestCase {
 
     public void testRangeQuery() {
@@ -49,7 +51,7 @@ public void testTermsQuery() {
         Mockito.when(context.indexVersionCreated()).thenReturn(IndexVersion.current());
         MappedFieldType ft = new ProvidedIdFieldMapper.IdFieldType(() -> false);
         Query query = ft.termQuery("id", context);
-        assertEquals(new TermInSetQuery("_id", Uid.encodeId("id")), query);
+        assertEquals(new TermInSetQuery("_id", List.of(Uid.encodeId("id"))), query);
     }
 
     public void testIsAggregatable() {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
index 205ff08c397b2..7a4ce8bcb03fa 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
@@ -901,6 +901,155 @@ public void testNestedArray() throws IOException {
         );
     }
 
+    public void testConflictingFieldNameAfterArray() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").startObject("properties");
+            {
+                b.startObject("to").startObject("properties");
+                {
+                    b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject();
+                }
+                b.endObject().endObject();
+                b.startObject("id").field("type", "float").endObject();
+            }
+            b.endObject().endObject();
+        })).documentMapper();
+
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("path");
+            {
+                b.startArray("to");
+                {
+                    b.startObject().array("id", 1, 20, 3).endObject();
+                    b.startObject().field("id", 10).endObject();
+                }
+                b.endArray();
+                b.field("id", "0.1");
+            }
+            b.endObject();
+        });
+        assertEquals("""
+            {"path":{"id":0.1,"to":{"id":[1,20,3,10]}}}""", syntheticSource);
+    }
+
+    public void testArrayWithNestedObjects() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").startObject("properties");
+            {
+                b.startObject("to").field("type", "nested").startObject("properties");
+                {
+                    b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject();
+                }
+                b.endObject().endObject();
+            }
+            b.endObject().endObject();
+        })).documentMapper();
+
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startArray("path");
+            {
+                b.startObject().startArray("to");
+                {
+                    b.startObject().array("id", 1, 20, 3).endObject();
+                    b.startObject().field("id", 10).endObject();
+                }
+                b.endArray().endObject();
+                b.startObject().startObject("to").field("id", "0.1").endObject().endObject();
+            }
+            b.endArray();
+        });
+        assertEquals("""
+            {"path":{"to":[{"id":[1,20,3]},{"id":10},{"id":0}]}}""", syntheticSource);
+    }
+
+    public void testObjectArrayWithinNestedObjects() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").startObject("properties");
+            {
+                b.startObject("to").field("type", "nested").startObject("properties");
+                {
+                    b.startObject("obj").startObject("properties");
+                    {
+                        b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject();
+                    }
+                    b.endObject().endObject();
+                }
+                b.endObject().endObject();
+            }
+            b.endObject().endObject();
+        })).documentMapper();
+
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("path");
+            {
+                b.startObject("to");
+                {
+                    b.startArray("obj");
+                    {
+                        b.startObject().array("id", 1, 20, 3).endObject();
+                        b.startObject().field("id", 10).endObject();
+                    }
+                    b.endArray();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        });
+        assertEquals("""
+            {"path":{"to":{"obj":[{"id":[1,20,3]},{"id":10}]}}}""", syntheticSource);
+    }
+
+    public void testObjectArrayWithinNestedObjectsArray() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").startObject("properties");
+            {
+                b.startObject("to").field("type", "nested").startObject("properties");
+                {
+                    b.startObject("obj").startObject("properties");
+                    {
+                        b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject();
+                    }
+                    b.endObject().endObject();
+                }
+                b.endObject().endObject();
+            }
+            b.endObject().endObject();
+        })).documentMapper();
+
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("path");
+            {
+                b.startArray("to");
+                {
+                    b.startObject();
+                    {
+                        b.startArray("obj");
+                        {
+                            b.startObject().array("id", 1, 20, 3).endObject();
+                            b.startObject().field("id", 10).endObject();
+                        }
+                        b.endArray();
+                    }
+                    b.endObject();
+                    b.startObject();
+                    {
+                        b.startArray("obj");
+                        {
+                            b.startObject().array("id", 200, 300, 500).endObject();
+                            b.startObject().field("id", 100).endObject();
+                        }
+                        b.endArray();
+                    }
+                    b.endObject();
+                }
+                b.endArray();
+            }
+            b.endObject();
+        });
+        assertEquals("""
+            {"path":{"to":[{"obj":[{"id":[1,20,3]},{"id":10}]},{"obj":[{"id":[200,300,500]},{"id":100}]}]}}""", syntheticSource);
+    }
+
     public void testArrayWithinArray() throws IOException {
         DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
             b.startObject("path");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtilTests.java
index 4170adf0a8508..8f209fb78fc64 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtilTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpPrefixAutomatonUtilTests.java
@@ -230,13 +230,8 @@ public void testAutomatonFromIPv6Group() throws UnknownHostException {
     }
 
     private static CompiledAutomaton compileAutomaton(Automaton automaton) {
-        CompiledAutomaton compiledAutomaton = new CompiledAutomaton(
-            automaton,
-            null,
-            false,
-            Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
-            true
-        );
+        automaton = Operations.determinize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton, false, false, true);
         return compiledAutomaton;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
index acbfe8b8f9b38..281d2993fa29c 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
@@ -125,16 +126,17 @@ public void testSort() throws IOException {
                 BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null);
                 SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false);
                 TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf));
+                StoredFields storedFields = reader.storedFields();
                 assertThat(
-                    reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(),
+                    storedFields.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(),
                     equalTo("{\"foo\": [\"192.168.0.1\"]}")
                 );
                 assertThat(
-                    reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(),
+                    storedFields.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(),
                     equalTo("{\"foo\": [\"192.168.0.2\"]}")
                 );
                 assertThat(
-                    reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(),
+                    storedFields.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(),
                     equalTo("{\"foo\": [\"192.168.0.4\"]}")
                 );
             }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
index b4c7ea0ed9508..e3bdb3d45818f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
@@ -110,7 +110,7 @@ protected TokenStream normalize(String fieldName, TokenStream in) {
 
     public void testTermsQuery() {
         MappedFieldType ft = new KeywordFieldType("field");
-        BytesRef[] terms = new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") };
+        List<BytesRef> terms = List.of(new BytesRef("foo"), new BytesRef("bar"));
         assertEquals(new TermInSetQuery("field", terms), ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_CONTEXT));
 
         MappedFieldType ft2 = new KeywordFieldType("field", false, true, Map.of());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
index f2e788918010c..57d52991a6442 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
@@ -115,9 +116,19 @@ public void testSort() throws IOException {
                 BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null);
                 SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false);
                 TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf));
-                assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"a\"]}"));
-                assertThat(reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"b\"]}"));
-                assertThat(reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"d\"]}"));
+                StoredFields storedFields = reader.storedFields();
+                assertThat(
+                    storedFields.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [\"a\"]}")
+                );
+                assertThat(
+                    storedFields.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [\"b\"]}")
+                );
+                assertThat(
+                    storedFields.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [\"d\"]}")
+                );
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
index 40357399cab5b..a8cb4d51c5efa 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.IndexSearcher;
@@ -132,9 +133,19 @@ public void testSort() throws IOException {
                 LongScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null);
                 SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false);
                 TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf));
-                assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1]}"));
-                assertThat(reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [2]}"));
-                assertThat(reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [4]}"));
+                StoredFields storedFields = reader.storedFields();
+                assertThat(
+                    storedFields.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [1]}")
+                );
+                assertThat(
+                    storedFields.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [2]}")
+                );
+                assertThat(
+                    storedFields.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(),
+                    equalTo("{\"foo\": [4]}")
+                );
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
index 64eee39532c31..3b77015fde415 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
@@ -169,27 +169,29 @@ public void testMergeEnabledForIndexTemplates() throws IOException {
         assertEquals(ObjectMapper.Subobjects.ENABLED, objectMapper.subobjects());
         assertTrue(objectMapper.sourceKeepMode().isEmpty());
 
-        // Setting 'enabled' to true is allowed, and updates the mapping.
-        update = Strings.toString(
-            XContentFactory.jsonBuilder()
-                .startObject()
-                .startObject("properties")
-                .startObject("object")
-                .field("type", "object")
-                .field("enabled", true)
-                .field("subobjects", "auto")
-                .field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true)
-                .endObject()
-                .endObject()
-                .endObject()
-        );
-        mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE);
-
-        objectMapper = mapper.mappers().objectMappers().get("object");
-        assertNotNull(objectMapper);
-        assertTrue(objectMapper.isEnabled());
-        assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects());
-        assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE));
+        if (ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()) {
+            // Setting 'enabled' to true is allowed, and updates the mapping.
+            update = Strings.toString(
+                XContentFactory.jsonBuilder()
+                    .startObject()
+                    .startObject("properties")
+                    .startObject("object")
+                    .field("type", "object")
+                    .field("enabled", true)
+                    .field("subobjects", "auto")
+                    .field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true)
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            );
+            mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE);
+
+            objectMapper = mapper.mappers().objectMappers().get("object");
+            assertNotNull(objectMapper);
+            assertTrue(objectMapper.isEnabled());
+            assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects());
+            assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE));
+        }
     }
 
     public void testFieldReplacementForIndexTemplates() throws IOException {
@@ -503,6 +505,7 @@ public void testSubobjectsCannotBeUpdatedOnRoot() throws IOException {
     }
 
     public void testSubobjectsAuto() throws Exception {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createMapperService(mapping(b -> {
             b.startObject("metrics.service");
             {
@@ -532,6 +535,7 @@ public void testSubobjectsAuto() throws Exception {
     }
 
     public void testSubobjectsAutoWithInnerObject() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createMapperService(mapping(b -> {
             b.startObject("metrics.service");
             {
@@ -565,6 +569,7 @@ public void testSubobjectsAutoWithInnerObject() throws IOException {
     }
 
     public void testSubobjectsAutoWithInnerNested() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createMapperService(mapping(b -> {
             b.startObject("metrics.service");
             {
@@ -586,6 +591,7 @@ public void testSubobjectsAutoWithInnerNested() throws IOException {
     }
 
     public void testSubobjectsAutoRoot() throws Exception {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createMapperService(mappingWithSubobjects(b -> {
             b.startObject("metrics.service.time");
             b.field("type", "long");
@@ -606,6 +612,7 @@ public void testSubobjectsAutoRoot() throws Exception {
     }
 
     public void testSubobjectsAutoRootWithInnerObject() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createMapperService(mappingWithSubobjects(b -> {
             b.startObject("metrics.service.time");
             {
@@ -626,6 +633,7 @@ public void testSubobjectsAutoRootWithInnerObject() throws IOException {
     }
 
     public void testSubobjectsAutoRootWithInnerNested() throws IOException {
+        assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createMapperService(mappingWithSubobjects(b -> {
             b.startObject("metrics.service");
             b.field("type", "nested");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index 2f417e688cb97..df6d9380fd141 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -443,6 +443,167 @@ public void testRecoverySourceWithLogs() throws IOException {
         }
     }
 
+    public void testStandardIndexModeWithSourceModeSetting() throws IOException {
+        // Test for IndexMode.STANDARD
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isSynthetic());
+        }
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            final DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isStored());
+        }
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            final DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isDisabled());
+        }
+
+        // Test for IndexMode.LOGSDB
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isSynthetic());
+        }
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            final DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isStored());
+        }
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .build();
+            var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
+            assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage());
+        }
+
+        // Test for IndexMode.TIME_SERIES
+        {
+            final String mappings = """
+                    {
+                        "_doc" : {
+                            "properties": {
+                                "routing_field": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        }
+                    }
+                """;
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isSynthetic());
+        }
+        {
+            final String mappings = """
+                    {
+                        "_doc" : {
+                            "properties": {
+                                "routing_field": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        }
+                    }
+                """;
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            final DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isStored());
+        }
+        {
+            final String mappings = """
+                    {
+                        "_doc" : {
+                            "properties": {
+                                "routing_field": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        }
+                    }
+                """;
+            final Settings settings = Settings.builder()
+                .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
+                .build();
+            var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
+            assertEquals("Failed to parse mapping: _source can not be disabled in index using [time_series] index mode", ex.getMessage());
+        }
+
+        // Test cases without IndexMode (default to standard)
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isSynthetic());
+        }
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            final DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isStored());
+        }
+        {
+            final XContentBuilder mappings = topMapping(b -> {});
+            final Settings settings = Settings.builder()
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .build();
+            final MapperService mapperService = createMapperService(settings, mappings);
+            final DocumentMapper docMapper = mapperService.documentMapper();
+            assertTrue(docMapper.sourceMapper().isDisabled());
+        }
+    }
+
     public void testRecoverySourceWithLogsCustom() throws IOException {
         XContentBuilder mappings = topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject());
         {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java
index 5360215b5b05b..836b791af23c1 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java
@@ -80,7 +80,7 @@ public void testBytesAndNumericRepresentation() throws Exception {
                 "field10"
             );
             CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(fieldNames, false);
-            searcher.doc(0, fieldsVisitor);
+            searcher.storedFields().document(0, fieldsVisitor);
 
             fieldsVisitor.postProcess(mapperService::fieldType);
             assertThat(fieldsVisitor.fields().size(), equalTo(10));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
index c8fcf486068c4..c2375e948fda0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
@@ -879,7 +879,7 @@ protected TokenStreamComponents createComponents(String fieldName) {
             IndexSearcher searcher = newSearcher(ir);
             MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder("field", "Prio 1");
             TopDocs td = searcher.search(queryBuilder.toQuery(searchExecutionContext), 1);
-            assertEquals(1, td.totalHits.value);
+            assertEquals(1, td.totalHits.value());
         });
 
         Exception e = expectThrows(
@@ -1353,6 +1353,7 @@ private void testBlockLoaderFromParent(boolean columnReader, boolean syntheticSo
         };
         MapperService mapper = createMapperService(syntheticSource ? syntheticSourceMapping(buildFields) : mapping(buildFields));
         BlockReaderSupport blockReaderSupport = getSupportedReaders(mapper, "field.sub");
-        testBlockLoader(columnReader, example, blockReaderSupport);
+        var sourceLoader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP);
+        testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java
index 5a34886d73db7..c8d7ad8127b55 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java
@@ -81,7 +81,7 @@ public void testTermQuery() {
     public void testTermsQuery() {
         KeyedFlattenedFieldType ft = createFieldType();
 
-        Query expected = new TermInSetQuery(ft.name(), new BytesRef("key\0value1"), new BytesRef("key\0value2"));
+        Query expected = new TermInSetQuery(ft.name(), List.of(new BytesRef("key\0value1"), new BytesRef("key\0value2")));
 
         List<String> terms = new ArrayList<>();
         terms.add("value1");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java
index f494af259c504..b52192d6e47b4 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java
@@ -23,8 +23,6 @@
 
 import java.io.IOException;
 
-import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
-
 public class KeyedFlattenedLeafFieldDataTests extends ESTestCase {
     private LeafOrdinalsFieldData delegate;
 
@@ -121,7 +119,8 @@ public void testNextOrd() throws IOException {
         docValues.advanceExact(0);
 
         int retrievedOrds = 0;
-        for (long ord = docValues.nextOrd(); ord != NO_MORE_ORDS; ord = docValues.nextOrd()) {
+        for (int i = 0; i < docValues.docValueCount(); i++) {
+            long ord = docValues.nextOrd();
             assertTrue(0 <= ord && ord < 10);
             retrievedOrds++;
 
@@ -190,9 +189,7 @@ public boolean advanceExact(int docID) {
 
         @Override
         public long nextOrd() {
-            if (index == documentOrds.length) {
-                return NO_MORE_ORDS;
-            }
+            assertTrue(index < documentOrds.length);
             return documentOrds[index++];
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValuesTests.java
index b2ffb779be00b..de4ab0bc5df30 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenormalizedCosineFloatVectorValuesTests.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.index.mapper.vectors;
 
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.NumericDocValues;
 import org.elasticsearch.test.ESTestCase;
 
@@ -25,7 +26,7 @@ public void testEmptyVectors() throws IOException {
             wrap(new float[0][0]),
             wrapMagnitudes(new float[0])
         );
-        assertEquals(NO_MORE_DOCS, normalizedCosineFloatVectorValues.nextDoc());
+        assertEquals(NO_MORE_DOCS, normalizedCosineFloatVectorValues.iterator().nextDoc());
     }
 
     public void testRandomVectors() throws IOException {
@@ -47,9 +48,10 @@ public void testRandomVectors() throws IOException {
             wrapMagnitudes(magnitudes)
         );
 
+        KnnVectorValues.DocIndexIterator iterator = normalizedCosineFloatVectorValues.iterator();
         for (int i = 0; i < numVectors; i++) {
-            assertEquals(i, normalizedCosineFloatVectorValues.advance(i));
-            assertArrayEquals(vectors[i], normalizedCosineFloatVectorValues.vectorValue(), (float) 1e-6);
+            assertEquals(i, iterator.advance(i));
+            assertArrayEquals(vectors[i], normalizedCosineFloatVectorValues.vectorValue(iterator.index()), (float) 1e-6);
             assertEquals(magnitudes[i], normalizedCosineFloatVectorValues.magnitude(), (float) 1e-6);
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
index 04b9b05ecfe3a..de084cd4582e2 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
@@ -1227,13 +1227,16 @@ public void testInvalidParameters() {
             e.getMessage(),
             containsString("Failed to parse mapping: Mapping definition for [field] has unsupported parameters:  [foo : {}]")
         );
-        for (String quantizationKind : new String[] { "int4_hnsw", "int8_hnsw", "int8_flat", "int4_flat" }) {
+        List<String> floatOnlyQuantizations = new ArrayList<>(
+            Arrays.asList("int4_hnsw", "int8_hnsw", "int8_flat", "int4_flat", "bbq_hnsw", "bbq_flat")
+        );
+        for (String quantizationKind : floatOnlyQuantizations) {
             e = expectThrows(
                 MapperParsingException.class,
                 () -> createDocumentMapper(
                     fieldMapping(
                         b -> b.field("type", "dense_vector")
-                            .field("dims", dims)
+                            .field("dims", 64)
                             .field("element_type", "byte")
                             .field("similarity", "l2_norm")
                             .field("index", true)
@@ -1939,6 +1942,60 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException {
         assertEquals(expectedString, knnVectorsFormat.toString());
     }
 
+    public void testKnnBBQHNSWVectorsFormat() throws IOException {
+        final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10);
+        final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10);
+        final int dims = randomIntBetween(64, 4096);
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            b.field("type", "dense_vector");
+            b.field("dims", dims);
+            b.field("index", true);
+            b.field("similarity", "dot_product");
+            b.startObject("index_options");
+            b.field("type", "bbq_hnsw");
+            b.field("m", m);
+            b.field("ef_construction", efConstruction);
+            b.endObject();
+        }));
+        CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE);
+        Codec codec = codecService.codec("default");
+        KnnVectorsFormat knnVectorsFormat;
+        if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
+            assertThat(codec, instanceOf(PerFieldMapperCodec.class));
+            knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
+        } else {
+            if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
+                codec = deduplicateFieldInfosCodec.delegate();
+            }
+            assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
+            knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
+        }
+        String expectedString = "ES816HnswBinaryQuantizedVectorsFormat(name=ES816HnswBinaryQuantizedVectorsFormat, maxConn="
+            + m
+            + ", beamWidth="
+            + efConstruction
+            + ", flatVectorFormat=ES816BinaryQuantizedVectorsFormat("
+            + "name=ES816BinaryQuantizedVectorsFormat, "
+            + "flatVectorScorer=ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())))";
+        assertEquals(expectedString, knnVectorsFormat.toString());
+    }
+
+    public void testInvalidVectorDimensionsBBQ() {
+        for (String quantizedFlatFormat : new String[] { "bbq_hnsw", "bbq_flat" }) {
+            MapperParsingException e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
+                b.field("type", "dense_vector");
+                b.field("dims", randomIntBetween(1, 63));
+                b.field("element_type", "float");
+                b.field("index", true);
+                b.field("similarity", "dot_product");
+                b.startObject("index_options");
+                b.field("type", quantizedFlatFormat);
+                b.endObject();
+            })));
+            assertThat(e.getMessage(), containsString("does not support dimensions fewer than 64"));
+        }
+    }
+
     public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException {
         final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10);
         final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10);
@@ -2022,24 +2079,27 @@ protected boolean supportsEmptyInputArray() {
 
     private static class DenseVectorSyntheticSourceSupport implements SyntheticSourceSupport {
         private final int dims = between(5, 1000);
-        private final ElementType elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT);
+        private final ElementType elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT);
         private final boolean indexed = randomBoolean();
         private final boolean indexOptionsSet = indexed && randomBoolean();
 
         @Override
         public SyntheticSourceExample example(int maxValues) throws IOException {
-            Object value = elementType == ElementType.BYTE
-                ? randomList(dims, dims, ESTestCase::randomByte)
-                : randomList(dims, dims, ESTestCase::randomFloat);
+            Object value = switch (elementType) {
+                case BYTE, BIT:
+                    yield randomList(dims, dims, ESTestCase::randomByte);
+                case FLOAT:
+                    yield randomList(dims, dims, ESTestCase::randomFloat);
+            };
             return new SyntheticSourceExample(value, value, this::mapping);
         }
 
         private void mapping(XContentBuilder b) throws IOException {
             b.field("type", "dense_vector");
-            b.field("dims", dims);
-            if (elementType == ElementType.BYTE || randomBoolean()) {
+            if (elementType == ElementType.BYTE || elementType == ElementType.BIT || randomBoolean()) {
                 b.field("element_type", elementType.toString());
             }
+            b.field("dims", elementType == ElementType.BIT ? dims * Byte.SIZE : dims);
             if (indexed) {
                 b.field("index", true);
                 b.field("similarity", "l2_norm");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
index 23864777db961..6433cf2f1c0d4 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -61,7 +62,9 @@ private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() {
             ),
             new DenseVectorFieldMapper.FlatIndexOptions(),
             new DenseVectorFieldMapper.Int8FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true))),
-            new DenseVectorFieldMapper.Int4FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)))
+            new DenseVectorFieldMapper.Int4FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true))),
+            new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)),
+            new DenseVectorFieldMapper.BBQFlatIndexOptions()
         );
     }
 
@@ -70,7 +73,7 @@ private DenseVectorFieldType createFloatFieldType() {
             "f",
             IndexVersion.current(),
             DenseVectorFieldMapper.ElementType.FLOAT,
-            6,
+            BBQ_MIN_DIMS,
             indexed,
             VectorSimilarity.COSINE,
             indexed ? randomIndexOptionsAll() : null,
@@ -147,7 +150,7 @@ public void testFetchSourceValue() throws IOException {
     public void testCreateNestedKnnQuery() {
         BitSetProducer producer = context -> null;
 
-        int dims = randomIntBetween(2, 2048);
+        int dims = randomIntBetween(BBQ_MIN_DIMS, 2048);
         if (dims % 2 != 0) {
             dims++;
         }
@@ -197,7 +200,7 @@ public void testCreateNestedKnnQuery() {
     }
 
     public void testExactKnnQuery() {
-        int dims = randomIntBetween(2, 2048);
+        int dims = randomIntBetween(BBQ_MIN_DIMS, 2048);
         if (dims % 2 != 0) {
             dims++;
         }
@@ -260,15 +263,19 @@ public void testFloatCreateKnnQuery() {
             "f",
             IndexVersion.current(),
             DenseVectorFieldMapper.ElementType.FLOAT,
-            4,
+            BBQ_MIN_DIMS,
             true,
             VectorSimilarity.DOT_PRODUCT,
             randomIndexOptionsAll(),
             Collections.emptyMap()
         );
+        float[] queryVector = new float[BBQ_MIN_DIMS];
+        for (int i = 0; i < BBQ_MIN_DIMS; i++) {
+            queryVector[i] = i;
+        }
         e = expectThrows(
             IllegalArgumentException.class,
-            () -> dotProductField.createKnnQuery(VectorData.fromFloats(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }), 10, 10, null, null, null)
+            () -> dotProductField.createKnnQuery(VectorData.fromFloats(queryVector), 10, 10, null, null, null)
         );
         assertThat(e.getMessage(), containsString("The [dot_product] similarity can only be used with unit-length vectors."));
 
@@ -276,7 +283,7 @@ public void testFloatCreateKnnQuery() {
             "f",
             IndexVersion.current(),
             DenseVectorFieldMapper.ElementType.FLOAT,
-            4,
+            BBQ_MIN_DIMS,
             true,
             VectorSimilarity.COSINE,
             randomIndexOptionsAll(),
@@ -284,7 +291,7 @@ public void testFloatCreateKnnQuery() {
         );
         e = expectThrows(
             IllegalArgumentException.class,
-            () -> cosineField.createKnnQuery(VectorData.fromFloats(new float[] { 0.0f, 0.0f, 0.0f, 0.0f }), 10, 10, null, null, null)
+            () -> cosineField.createKnnQuery(VectorData.fromFloats(new float[BBQ_MIN_DIMS]), 10, 10, null, null, null)
         );
         assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude."));
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java
index c007156c806eb..baade683a90fd 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java
@@ -208,7 +208,41 @@ public int size() {
             }
 
             @Override
-            public byte[] vectorValue() {
+            public DocIndexIterator iterator() {
+                return new DocIndexIterator() {
+                    @Override
+                    public int index() {
+                        return index;
+                    }
+
+                    @Override
+                    public int docID() {
+                        return index;
+                    }
+
+                    @Override
+                    public int nextDoc() {
+                        throw new UnsupportedOperationException();
+                    }
+
+                    @Override
+                    public int advance(int target) {
+                        if (target >= size()) {
+                            return NO_MORE_DOCS;
+                        }
+                        return index = target;
+                    }
+
+                    @Override
+                    public long cost() {
+                        return 0;
+                    }
+                };
+            }
+
+            @Override
+            public byte[] vectorValue(int ord) {
+                assert ord == index;
                 for (int i = 0; i < byteVector.length; i++) {
                     byteVector[i] = (byte) vectors[index][i];
                 }
@@ -216,25 +250,12 @@ public byte[] vectorValue() {
             }
 
             @Override
-            public int docID() {
-                return index;
-            }
-
-            @Override
-            public int nextDoc() {
+            public ByteVectorValues copy() {
                 throw new UnsupportedOperationException();
             }
 
             @Override
-            public int advance(int target) {
-                if (target >= size()) {
-                    return NO_MORE_DOCS;
-                }
-                return index = target;
-            }
-
-            @Override
-            public VectorScorer scorer(byte[] floats) throws IOException {
+            public VectorScorer scorer(byte[] floats) {
                 throw new UnsupportedOperationException();
             }
         };
@@ -256,30 +277,51 @@ public int size() {
             }
 
             @Override
-            public float[] vectorValue() {
-                return vectors[index];
-            }
-
-            @Override
-            public int docID() {
-                return index;
+            public DocIndexIterator iterator() {
+                return new DocIndexIterator() {
+                    @Override
+                    public int index() {
+                        return index;
+                    }
+
+                    @Override
+                    public int docID() {
+                        return index;
+                    }
+
+                    @Override
+                    public int nextDoc() throws IOException {
+                        return advance(index + 1);
+                    }
+
+                    @Override
+                    public int advance(int target) throws IOException {
+                        if (target >= size()) {
+                            return NO_MORE_DOCS;
+                        }
+                        return index = target;
+                    }
+
+                    @Override
+                    public long cost() {
+                        return 0;
+                    }
+                };
             }
 
             @Override
-            public int nextDoc() {
-                return advance(index + 1);
+            public float[] vectorValue(int ord) {
+                assert ord == index;
+                return vectors[index];
             }
 
             @Override
-            public int advance(int target) {
-                if (target >= size()) {
-                    return NO_MORE_DOCS;
-                }
-                return index = target;
+            public FloatVectorValues copy() {
+                throw new UnsupportedOperationException();
             }
 
             @Override
-            public VectorScorer scorer(float[] floats) throws IOException {
+            public VectorScorer scorer(float[] floats) {
                 throw new UnsupportedOperationException();
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index 25d4c1008ba7d..e9ef3ac8ad748 100644
--- a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -207,15 +207,15 @@ public void testMinShouldMatchFilterWithoutShouldClauses() throws Exception {
         assertThat(booleanQuery.getMinimumNumberShouldMatch(), equalTo(0));
         assertThat(booleanQuery.clauses().size(), equalTo(1));
         BooleanClause booleanClause = booleanQuery.clauses().get(0);
-        assertThat(booleanClause.getOccur(), equalTo(BooleanClause.Occur.FILTER));
-        assertThat(booleanClause.getQuery(), instanceOf(BooleanQuery.class));
-        BooleanQuery innerBooleanQuery = (BooleanQuery) booleanClause.getQuery();
+        assertThat(booleanClause.occur(), equalTo(BooleanClause.Occur.FILTER));
+        assertThat(booleanClause.query(), instanceOf(BooleanQuery.class));
+        BooleanQuery innerBooleanQuery = (BooleanQuery) booleanClause.query();
         // we didn't set minimum should match initially, there are no should clauses so it should be 0
         assertThat(innerBooleanQuery.getMinimumNumberShouldMatch(), equalTo(0));
         assertThat(innerBooleanQuery.clauses().size(), equalTo(1));
         BooleanClause innerBooleanClause = innerBooleanQuery.clauses().get(0);
-        assertThat(innerBooleanClause.getOccur(), equalTo(BooleanClause.Occur.MUST));
-        assertThat(innerBooleanClause.getQuery(), instanceOf(MatchAllDocsQuery.class));
+        assertThat(innerBooleanClause.occur(), equalTo(BooleanClause.Occur.MUST));
+        assertThat(innerBooleanClause.query(), instanceOf(MatchAllDocsQuery.class));
     }
 
     public void testMinShouldMatchBiggerThanNumberOfShouldClauses() throws Exception {
@@ -449,6 +449,12 @@ public void testRewriteWithMatchNone() throws IOException {
         rewritten = Rewriteable.rewrite(boolQueryBuilder, createSearchExecutionContext());
         assertNotEquals(new MatchNoneQueryBuilder(), rewritten);
 
+        boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.should(new WrapperQueryBuilder(new MatchNoneQueryBuilder().toString()));
+        boolQueryBuilder.mustNot(new TermQueryBuilder(TEXT_FIELD_NAME, "bar"));
+        rewritten = Rewriteable.rewrite(boolQueryBuilder, createSearchExecutionContext());
+        assertNotEquals(new MatchNoneQueryBuilder(), rewritten);
+
         boolQueryBuilder = new BoolQueryBuilder();
         boolQueryBuilder.filter(new TermQueryBuilder(TEXT_FIELD_NAME, "bar"));
         boolQueryBuilder.mustNot(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()).toString()));
diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java
index daaa0c4653d7a..f09e524faf8ff 100644
--- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryParsingTests.java
@@ -143,8 +143,8 @@ public void testWildcardFieldPattern() throws Exception {
 
             BooleanQuery booleanQuery = (BooleanQuery) query;
             assertThat(booleanQuery.clauses().size(), equalTo(2));
-            assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(CombinedFieldQuery.class));
-            assertThat(booleanQuery.clauses().get(1).getQuery(), instanceOf(CombinedFieldQuery.class));
+            assertThat(booleanQuery.clauses().get(0).query(), instanceOf(CombinedFieldQuery.class));
+            assertThat(booleanQuery.clauses().get(1).query(), instanceOf(CombinedFieldQuery.class));
         });
     }
 
@@ -164,8 +164,8 @@ public void testOperator() throws Exception {
         assertThat(booleanQuery.getMinimumNumberShouldMatch(), equalTo(minimumShouldMatch));
 
         assertThat(booleanQuery.clauses().size(), equalTo(2));
-        assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(occur));
-        assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(occur));
+        assertThat(booleanQuery.clauses().get(0).occur(), equalTo(occur));
+        assertThat(booleanQuery.clauses().get(1).occur(), equalTo(occur));
     }
 
     public void testQueryBoost() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java
index 49c646d243a55..ef5088eef84a7 100644
--- a/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java
@@ -10,7 +10,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.document.LatLonPoint;
-import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
@@ -81,7 +81,7 @@ protected void doAssertLuceneQuery(DistanceFeatureQueryBuilder queryBuilder, Que
             } else { // NANOSECONDS
                 pivotLong = pivotVal.getNanos();
             }
-            expectedQuery = LongPoint.newDistanceFeatureQuery(fieldName, 1.0f, originLong, pivotLong);
+            expectedQuery = LongField.newDistanceFeatureQuery(fieldName, 1.0f, originLong, pivotLong);
         }
         assertEquals(expectedQuery, query);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
index ba3350bca8e2c..afa8fc1529604 100644
--- a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
@@ -68,7 +68,7 @@ protected void doAssertLuceneQuery(ExistsQueryBuilder queryBuilder, Query query,
                 Collection<String> childFields = context.getMatchingFieldNames(field + ".*");
                 assertThat(booleanQuery.clauses().size(), equalTo(childFields.size()));
                 for (BooleanClause booleanClause : booleanQuery) {
-                    assertThat(booleanClause.getOccur(), equalTo(BooleanClause.Occur.SHOULD));
+                    assertThat(booleanClause.occur(), equalTo(BooleanClause.Occur.SHOULD));
                 }
             } else if (context.getFieldType(field).hasDocValues() || context.getFieldType(field).getTextSearchInfo().hasNorms()) {
                 assertThat(constantScoreQuery.getQuery(), instanceOf(FieldExistsQuery.class));
@@ -87,7 +87,7 @@ protected void doAssertLuceneQuery(ExistsQueryBuilder queryBuilder, Query query,
             assertThat(booleanQuery.clauses().size(), equalTo(fields.size()));
             for (int i = 0; i < fields.size(); i++) {
                 BooleanClause booleanClause = booleanQuery.clauses().get(i);
-                assertThat(booleanClause.getOccur(), equalTo(BooleanClause.Occur.SHOULD));
+                assertThat(booleanClause.occur(), equalTo(BooleanClause.Occur.SHOULD));
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java
index e471858ce9c5a..79f2dcb61e508 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilderTests.java
@@ -37,7 +37,6 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
-import static org.hamcrest.Matchers.hasProperty;
 import static org.hamcrest.Matchers.hasSize;
 
 public class MatchBoolPrefixQueryBuilderTests extends AbstractQueryTestCase<MatchBoolPrefixQueryBuilder> {
@@ -101,7 +100,7 @@ protected void doAssertLuceneQuery(MatchBoolPrefixQueryBuilder queryBuilder, Que
             // all queries except the last should be TermQuery or SynonymQuery
             final Set<Query> allQueriesExceptLast = IntStream.range(0, booleanQuery.clauses().size() - 1)
                 .mapToObj(booleanQuery.clauses()::get)
-                .map(BooleanClause::getQuery)
+                .map(BooleanClause::query)
                 .collect(Collectors.toSet());
             assertThat(
                 allQueriesExceptLast,
@@ -122,13 +121,13 @@ protected void doAssertLuceneQuery(MatchBoolPrefixQueryBuilder queryBuilder, Que
             });
 
             // the last query should be PrefixQuery
-            final Query shouldBePrefixQuery = booleanQuery.clauses().get(booleanQuery.clauses().size() - 1).getQuery();
+            final Query shouldBePrefixQuery = booleanQuery.clauses().get(booleanQuery.clauses().size() - 1).query();
             assertThat(shouldBePrefixQuery, instanceOf(PrefixQuery.class));
 
             if (queryBuilder.minimumShouldMatch() != null) {
                 final int optionalClauses = (int) booleanQuery.clauses()
                     .stream()
-                    .filter(clause -> clause.getOccur() == BooleanClause.Occur.SHOULD)
+                    .filter(clause -> clause.occur() == BooleanClause.Occur.SHOULD)
                     .count();
                 final int expected = Queries.calculateMinShouldMatch(optionalClauses, queryBuilder.minimumShouldMatch());
                 assertThat(booleanQuery.getMinimumNumberShouldMatch(), equalTo(expected));
@@ -266,10 +265,12 @@ private static void assertBooleanQuery(Query actual, List<Query> expectedClauseQ
         assertThat(actual, instanceOf(BooleanQuery.class));
         final BooleanQuery actualBooleanQuery = (BooleanQuery) actual;
         assertThat(actualBooleanQuery.clauses(), hasSize(expectedClauseQueries.size()));
-        assertThat(actualBooleanQuery.clauses(), everyItem(hasProperty("occur", equalTo(BooleanClause.Occur.SHOULD))));
 
         for (int i = 0; i < actualBooleanQuery.clauses().size(); i++) {
-            final Query clauseQuery = actualBooleanQuery.clauses().get(i).getQuery();
+            BooleanClause clause = actualBooleanQuery.clauses().get(i);
+            assertEquals(BooleanClause.Occur.SHOULD, clause.occur());
+            final Query clauseQuery = clause.query();
+
             assertThat(clauseQuery, equalTo(expectedClauseQueries.get(i)));
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index e71485647913c..ba46bf76efbfe 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -18,6 +18,7 @@
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -163,7 +164,7 @@ protected void doAssertLuceneQuery(MatchQueryBuilder queryBuilder, Query query,
                 // calculate expected minimumShouldMatch value
                 int optionalClauses = 0;
                 for (BooleanClause c : bq.clauses()) {
-                    if (c.getOccur() == BooleanClause.Occur.SHOULD) {
+                    if (c.occur() == BooleanClause.Occur.SHOULD) {
                         optionalClauses++;
                     }
                 }
@@ -527,9 +528,9 @@ public void testAliasWithSynonyms() throws Exception {
     public void testMaxBooleanClause() {
         MatchQueryParser query = new MatchQueryParser(createSearchExecutionContext());
         query.setAnalyzer(new MockGraphAnalyzer(createGiantGraph(40)));
-        expectThrows(BooleanQuery.TooManyClauses.class, () -> query.parse(Type.PHRASE, TEXT_FIELD_NAME, ""));
+        expectThrows(IndexSearcher.TooManyClauses.class, () -> query.parse(Type.PHRASE, TEXT_FIELD_NAME, ""));
         query.setAnalyzer(new MockGraphAnalyzer(createGiantGraphMultiTerms()));
-        expectThrows(BooleanQuery.TooManyClauses.class, () -> query.parse(Type.PHRASE, TEXT_FIELD_NAME, ""));
+        expectThrows(IndexSearcher.TooManyClauses.class, () -> query.parse(Type.PHRASE, TEXT_FIELD_NAME, ""));
     }
 
     private static class MockGraphAnalyzer extends Analyzer {
@@ -567,7 +568,7 @@ private static CannedBinaryTokenStream.BinaryToken[] createGiantGraph(int numPos
     }
 
     /**
-     * Creates a graph token stream with {@link BooleanQuery#getMaxClauseCount()}
+     * Creates a graph token stream with {@link IndexSearcher#getMaxClauseCount()}
      * expansions at the last position.
      **/
     private static CannedBinaryTokenStream.BinaryToken[] createGiantGraphMultiTerms() {
@@ -578,7 +579,7 @@ private static CannedBinaryTokenStream.BinaryToken[] createGiantGraphMultiTerms(
         tokens.add(new CannedBinaryTokenStream.BinaryToken(term1, 0, 2));
         tokens.add(new CannedBinaryTokenStream.BinaryToken(term2, 1, 1));
         tokens.add(new CannedBinaryTokenStream.BinaryToken(term2, 1, 1));
-        for (int i = 0; i < BooleanQuery.getMaxClauseCount(); i++) {
+        for (int i = 0; i < IndexSearcher.getMaxClauseCount(); i++) {
             tokens.add(new CannedBinaryTokenStream.BinaryToken(term1, 0, 1));
         }
         return tokens.toArray(new CannedBinaryTokenStream.BinaryToken[0]);
diff --git a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index 7209ee77cb70d..7c21751b4b332 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -246,7 +246,7 @@ private static Fields generateFields(String[] fieldNames, String text) throws IO
         for (String fieldName : fieldNames) {
             index.addField(fieldName, text, new WhitespaceAnalyzer());
         }
-        return index.createSearcher().getIndexReader().getTermVectors(0);
+        return index.createSearcher().getIndexReader().termVectors().get(0);
     }
 
     @Override
@@ -255,7 +255,7 @@ protected void doAssertLuceneQuery(MoreLikeThisQueryBuilder queryBuilder, Query
             assertThat(query, instanceOf(BooleanQuery.class));
             BooleanQuery booleanQuery = (BooleanQuery) query;
             for (BooleanClause booleanClause : booleanQuery) {
-                if (booleanClause.getQuery() instanceof MoreLikeThisQuery moreLikeThisQuery) {
+                if (booleanClause.query() instanceof MoreLikeThisQuery moreLikeThisQuery) {
                     assertThat(moreLikeThisQuery.getLikeFields().length, greaterThan(0));
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index 0260fa2ef4cc8..918815f2a4f77 100644
--- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -17,7 +17,9 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.test.AbstractQueryTestCase;
+import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -175,4 +177,37 @@ public void testMustRewrite() throws IOException {
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> queryBuilder.toQuery(context));
         assertEquals("Rewrite first", e.getMessage());
     }
+
+    public void testCoordinatorTierRewriteToMatchAll() throws IOException {
+        QueryBuilder query = new PrefixQueryBuilder("_tier", "data_fro");
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchAllQueryBuilder.class));
+    }
+
+    public void testCoordinatorTierRewriteToMatchNone() throws IOException {
+        QueryBuilder query = QueryBuilders.boolQuery().mustNot(new PrefixQueryBuilder("_tier", "data_fro"));
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchNoneQueryBuilder.class));
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java
new file mode 100644
index 0000000000000..0b2a8ab4856b3
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.indices.DateFieldRangeInfo;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class QueryRewriteContextTests extends ESTestCase {
+
+    public void testGetTierPreference() {
+        {
+            // cold->hot tier preference
+            IndexMetadata metadata = newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                    .put(DataTier.TIER_PREFERENCE, "data_cold,data_warm,data_hot")
+                    .build()
+            );
+            QueryRewriteContext context = new QueryRewriteContext(
+                parserConfig(),
+                null,
+                System::currentTimeMillis,
+                null,
+                MappingLookup.EMPTY,
+                Collections.emptyMap(),
+                new IndexSettings(metadata, Settings.EMPTY),
+                null,
+                null,
+                null,
+                null,
+                null,
+                null,
+                null,
+                null
+            );
+
+            assertThat(context.getTierPreference(), is("data_cold"));
+        }
+
+        {
+            // missing tier preference
+            IndexMetadata metadata = newIndexMeta(
+                "index",
+                Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()
+            );
+            QueryRewriteContext context = new QueryRewriteContext(
+                parserConfig(),
+                null,
+                System::currentTimeMillis,
+                null,
+                MappingLookup.EMPTY,
+                Collections.emptyMap(),
+                new IndexSettings(metadata, Settings.EMPTY),
+                null,
+                null,
+                null,
+                null,
+                null,
+                null,
+                null,
+                null
+            );
+
+            assertThat(context.getTierPreference(), is(nullValue()));
+        }
+
+        {
+            // coordinator rewrite context
+            IndexMetadata metadata = newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                    .put(DataTier.TIER_PREFERENCE, "data_cold,data_warm,data_hot")
+                    .build()
+            );
+            CoordinatorRewriteContext coordinatorRewriteContext = new CoordinatorRewriteContext(
+                parserConfig(),
+                null,
+                System::currentTimeMillis,
+                new DateFieldRangeInfo(null, null, new DateFieldMapper.DateFieldType(IndexMetadata.EVENT_INGESTED_FIELD_NAME), null),
+                "data_frozen"
+            );
+
+            assertThat(coordinatorRewriteContext.getTierPreference(), is("data_frozen"));
+        }
+        {
+            // coordinator rewrite context empty tier
+            IndexMetadata metadata = newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                    .put(DataTier.TIER_PREFERENCE, "data_cold,data_warm,data_hot")
+                    .build()
+            );
+            CoordinatorRewriteContext coordinatorRewriteContext = new CoordinatorRewriteContext(
+                parserConfig(),
+                null,
+                System::currentTimeMillis,
+                new DateFieldRangeInfo(null, null, new DateFieldMapper.DateFieldType(IndexMetadata.EVENT_INGESTED_FIELD_NAME), null),
+                ""
+            );
+
+            assertThat(coordinatorRewriteContext.getTierPreference(), is(nullValue()));
+        }
+    }
+
+    public static IndexMetadata newIndexMeta(String name, Settings indexSettings) {
+        return IndexMetadata.builder(name).settings(indexSettings(IndexVersion.current(), 1, 1).put(indexSettings)).build();
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index f4405229e857e..3dcf00e4f22f5 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -749,7 +749,7 @@ public void testToQueryRegExpQueryTooComplex() throws Exception {
             TooComplexToDeterminizeException.class,
             () -> queryBuilder.toQuery(createSearchExecutionContext())
         );
-        assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
+        assertThat(e.getMessage(), containsString("Determinizing automaton"));
         assertThat(e.getMessage(), containsString("would require more than 10000 effort."));
     }
 
@@ -775,7 +775,7 @@ public void testToQueryRegExpQueryMaxDeterminizedStatesParsing() throws Exceptio
             TooComplexToDeterminizeException.class,
             () -> queryBuilder.toQuery(createSearchExecutionContext())
         );
-        assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
+        assertThat(e.getMessage(), containsString("Determinizing automaton"));
         assertThat(e.getMessage(), containsString("would require more than 10 effort."));
     }
 
@@ -925,10 +925,10 @@ public void testToQueryBooleanQueryMultipleBoosts() throws Exception {
         assertThat(query, instanceOf(BooleanQuery.class));
         BooleanQuery booleanQuery = (BooleanQuery) query;
         assertThat(booleanQuery.getMinimumNumberShouldMatch(), equalTo(2));
-        assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
-        assertThat(booleanQuery.clauses().get(0).getQuery(), equalTo(new TermQuery(new Term(TEXT_FIELD_NAME, "foo"))));
-        assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
-        assertThat(booleanQuery.clauses().get(1).getQuery(), equalTo(new TermQuery(new Term(TEXT_FIELD_NAME, "bar"))));
+        assertThat(booleanQuery.clauses().get(0).occur(), equalTo(BooleanClause.Occur.SHOULD));
+        assertThat(booleanQuery.clauses().get(0).query(), equalTo(new TermQuery(new Term(TEXT_FIELD_NAME, "foo"))));
+        assertThat(booleanQuery.clauses().get(1).occur(), equalTo(BooleanClause.Occur.SHOULD));
+        assertThat(booleanQuery.clauses().get(1).query(), equalTo(new TermQuery(new Term(TEXT_FIELD_NAME, "bar"))));
     }
 
     public void testToQueryPhraseQueryBoostAndSlop() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
index 658176b1d31dd..fdc18264e2299 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
@@ -384,7 +384,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() {
 
     public void testSyntheticSourceSearchLookup() throws IOException {
         // Build a mapping using synthetic source
-        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true).setSynthetic().build();
+        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false).setSynthetic().build();
         RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100)
         ).build(MapperBuilderContext.root(true, false));
diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index 79c4c9ec5bb20..a84cd60a99e45 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -315,7 +315,7 @@ protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query
     private static int shouldClauses(BooleanQuery query) {
         int result = 0;
         for (BooleanClause c : query.clauses()) {
-            if (c.getOccur() == BooleanClause.Occur.SHOULD) {
+            if (c.occur() == BooleanClause.Occur.SHOULD) {
                 result++;
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
index cb314472e35b3..283bbbc9b100d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -18,7 +18,6 @@
 import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.queries.spans.SpanTermQuery;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiTermQuery;
@@ -220,7 +219,7 @@ public void testToQueryInnerTermQuery() throws IOException {
             assertThat(prefixQuery.getPrefix().text(), equalTo("f"));
             assertThat(wrapper.getRewriteMethod(), instanceOf(SpanBooleanQueryRewriteWithMaxClause.class));
             SpanBooleanQueryRewriteWithMaxClause rewrite = (SpanBooleanQueryRewriteWithMaxClause) wrapper.getRewriteMethod();
-            assertThat(rewrite.getMaxExpansions(), equalTo(BooleanQuery.getMaxClauseCount()));
+            assertThat(rewrite.getMaxExpansions(), equalTo(IndexSearcher.getMaxClauseCount()));
             assertTrue(rewrite.isHardLimit());
         }
     }
@@ -265,8 +264,8 @@ public void testTermExpansionExceptionOnSpanFailure() throws Exception {
                     iw.addDocument(singleton(new TextField("body", "foo bar" + Integer.toString(i), Field.Store.NO)));
                 }
                 try (IndexReader reader = iw.getReader()) {
-                    int origBoolMaxClauseCount = BooleanQuery.getMaxClauseCount();
-                    BooleanQuery.setMaxClauseCount(1);
+                    int origBoolMaxClauseCount = IndexSearcher.getMaxClauseCount();
+                    IndexSearcher.setMaxClauseCount(1);
                     try {
                         QueryBuilder queryBuilder = new SpanMultiTermQueryBuilder(QueryBuilders.prefixQuery("body", "bar"));
                         IndexSearcher searcher = newSearcher(reader);
@@ -274,7 +273,7 @@ public void testTermExpansionExceptionOnSpanFailure() throws Exception {
                         RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(searcher));
                         assertThat(exc.getMessage(), containsString("maxClauseCount"));
                     } finally {
-                        BooleanQuery.setMaxClauseCount(origBoolMaxClauseCount);
+                        IndexSearcher.setMaxClauseCount(origBoolMaxClauseCount);
                     }
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
index b5cf42cf5df28..bbac216754eed 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
@@ -17,9 +17,11 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.xcontent.json.JsonStringEncoder;
+import org.hamcrest.CoreMatchers;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -238,4 +240,36 @@ public void testLongTerm() throws IOException {
             { "term" : { "foo" : "%s" } }""", longTerm)));
         assertThat(e.getMessage(), containsString("term starting with [aaaaa"));
     }
+
+    public void testCoordinatorTierRewriteToMatchAll() throws IOException {
+        QueryBuilder query = new TermQueryBuilder("_tier", "data_frozen");
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchAllQueryBuilder.class));
+    }
+
+    public void testCoordinatorTierRewriteToMatchNone() throws IOException {
+        QueryBuilder query = QueryBuilders.boolQuery().mustNot(new TermQueryBuilder("_tier", "data_frozen"));
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchNoneQueryBuilder.class));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index 1ce69355379de..2faee7bc89eb5 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -317,6 +318,38 @@ public void testLongTerm() throws IOException {
         assertThat(e.getMessage(), containsString("term starting with [aaaaa"));
     }
 
+    public void testCoordinatorTierRewriteToMatchAll() throws IOException {
+        QueryBuilder query = new TermsQueryBuilder("_tier", "data_frozen");
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchAllQueryBuilder.class));
+    }
+
+    public void testCoordinatorTierRewriteToMatchNone() throws IOException {
+        QueryBuilder query = QueryBuilders.boolQuery().mustNot(new TermsQueryBuilder("_tier", "data_frozen"));
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchNoneQueryBuilder.class));
+    }
+
     @Override
     protected QueryBuilder parseQuery(XContentParser parser) throws IOException {
         QueryBuilder query = super.parseQuery(parser);
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
index 3edf150688384..589019093075d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
@@ -238,7 +238,7 @@ public void testDoToQuery() throws Exception {
                     10,
                     new Sort(SortField.FIELD_DOC)
                 );
-                assertThat(topDocsWithMinimumShouldMatchField.totalHits.value, equalTo(3L));
+                assertThat(topDocsWithMinimumShouldMatchField.totalHits.value(), equalTo(3L));
                 assertThat(topDocsWithMinimumShouldMatchField.scoreDocs[0].doc, equalTo(1));
                 assertThat(topDocsWithMinimumShouldMatchField.scoreDocs[1].doc, equalTo(3));
                 assertThat(topDocsWithMinimumShouldMatchField.scoreDocs[2].doc, equalTo(4));
@@ -249,7 +249,7 @@ public void testDoToQuery() throws Exception {
                 ).doToQuery(context);
                 searcher = newSearcher(ir);
                 TopDocs topDocsWithMinimumShouldMatch = searcher.search(queryWithMinimumShouldMatch, 10, new Sort(SortField.FIELD_DOC));
-                assertThat(topDocsWithMinimumShouldMatch.totalHits.value, equalTo(5L));
+                assertThat(topDocsWithMinimumShouldMatch.totalHits.value(), equalTo(5L));
                 assertThat(topDocsWithMinimumShouldMatch.scoreDocs[0].doc, equalTo(1));
                 assertThat(topDocsWithMinimumShouldMatch.scoreDocs[1].doc, equalTo(2));
                 assertThat(topDocsWithMinimumShouldMatch.scoreDocs[2].doc, equalTo(3));
@@ -266,7 +266,7 @@ public void testDoToQuery() throws Exception {
                     10,
                     new Sort(SortField.FIELD_DOC)
                 );
-                assertThat(topDocsWithMinimumShouldMatchNegative.totalHits.value, equalTo(1L));
+                assertThat(topDocsWithMinimumShouldMatchNegative.totalHits.value(), equalTo(1L));
                 assertThat(topDocsWithMinimumShouldMatchNegative.scoreDocs[0].doc, equalTo(5));
             }
         }
@@ -310,7 +310,7 @@ public void testDoToQuery_msmScriptField() throws Exception {
                     .doToQuery(context);
                 IndexSearcher searcher = newSearcher(ir);
                 TopDocs topDocs = searcher.search(query, 10, new Sort(SortField.FIELD_DOC));
-                assertThat(topDocs.totalHits.value, equalTo(3L));
+                assertThat(topDocs.totalHits.value(), equalTo(3L));
                 assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
                 assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
                 assertThat(topDocs.scoreDocs[2].doc, equalTo(4));
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 7ee6d75a08736..182bd4d6b5b86 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -15,7 +15,9 @@
 import org.apache.lucene.search.WildcardQuery;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.test.AbstractQueryTestCase;
+import org.hamcrest.CoreMatchers;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -166,4 +168,36 @@ public void testMustRewrite() throws IOException {
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> queryBuilder.toQuery(context));
         assertEquals("Rewrite first", e.getMessage());
     }
+
+    public void testCoordinatorTierRewriteToMatchAll() throws IOException {
+        QueryBuilder query = new WildcardQueryBuilder("_tier", "data_fr*");
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchAllQueryBuilder.class));
+    }
+
+    public void testCoordinatorTierRewriteToMatchNone() throws IOException {
+        QueryBuilder query = QueryBuilders.boolQuery().mustNot(new WildcardQueryBuilder("_tier", "data_fro*"));
+        final String timestampFieldName = "@timestamp";
+        long minTimestamp = 1685714000000L;
+        long maxTimestamp = 1685715000000L;
+        final CoordinatorRewriteContext coordinatorRewriteContext = createCoordinatorRewriteContext(
+            new DateFieldMapper.DateFieldType(timestampFieldName),
+            minTimestamp,
+            maxTimestamp,
+            "data_frozen"
+        );
+
+        QueryBuilder rewritten = query.rewrite(coordinatorRewriteContext);
+        assertThat(rewritten, CoreMatchers.instanceOf(MatchNoneQueryBuilder.class));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
index 47c75ee38ee1b..49b1362436ec7 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -305,7 +305,7 @@ public void testConflictingOpsOnReplica() throws Exception {
             for (IndexShard shard : shards) {
                 try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
                     TopDocs search = searcher.search(new TermQuery(new Term("f", "2")), 10);
-                    assertEquals("shard " + shard.routingEntry() + " misses new version", 1, search.totalHits.value);
+                    assertEquals("shard " + shard.routingEntry() + " misses new version", 1, search.totalHits.value());
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java b/server/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java
index 5308f5f5d1f04..e8652e3a0f6d6 100644
--- a/server/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTestCase.java
@@ -225,7 +225,7 @@ public void testNestedSorting() throws Exception {
 
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopFieldDocs topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(7L));
+        assertThat(topDocs.totalHits.value(), equalTo(7L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(11));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(7));
@@ -240,7 +240,7 @@ public void testNestedSorting() throws Exception {
 
         sort = new Sort(new SortField("field2", nestedComparatorSource, true));
         topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(7L));
+        assertThat(topDocs.totalHits.value(), equalTo(7L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(28));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(13));
@@ -258,7 +258,7 @@ public void testNestedSorting() throws Exception {
         query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None);
         sort = new Sort(new SortField("field2", nestedComparatorSource, true));
         topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(6L));
+        assertThat(topDocs.totalHits.value(), equalTo(6L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(23));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(12));
@@ -273,7 +273,7 @@ public void testNestedSorting() throws Exception {
 
         sort = new Sort(new SortField("field2", nestedComparatorSource));
         topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(6L));
+        assertThat(topDocs.totalHits.value(), equalTo(6L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(15));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(3));
@@ -289,7 +289,7 @@ public void testNestedSorting() throws Exception {
         nestedComparatorSource = createFieldComparator("field2", sortMode, 127, createNested(searcher, parentFilter, childFilter));
         sort = new Sort(new SortField("field2", nestedComparatorSource, true));
         topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(8L));
+        assertThat(topDocs.totalHits.value(), equalTo(8L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(19));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(127));
@@ -305,7 +305,7 @@ public void testNestedSorting() throws Exception {
         nestedComparatorSource = createFieldComparator("field2", sortMode, -127, createNested(searcher, parentFilter, childFilter));
         sort = new Sort(new SortField("field2", nestedComparatorSource));
         topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(8L));
+        assertThat(topDocs.totalHits.value(), equalTo(8L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(19));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(-127));
@@ -340,7 +340,7 @@ protected void assertAvgScoreMode(Query parentFilter, IndexSearcher searcher) th
         );
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopDocs topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(7L));
+        assertThat(topDocs.totalHits.value(), equalTo(7L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(11));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(2));
diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java
index cecf20360178c..ca176a5402c06 100644
--- a/server/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java
@@ -73,7 +73,7 @@ protected void assertAvgScoreMode(Query parentFilter, IndexSearcher searcher) th
         );
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopDocs topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(7L));
+        assertThat(topDocs.totalHits.value(), equalTo(7L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(11));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(2));
diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java
index 49d9c0eedd121..60e7473a2101a 100644
--- a/server/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java
@@ -76,7 +76,7 @@ protected void assertAvgScoreMode(
         );
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopDocs topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(7L));
+        assertThat(topDocs.totalHits.value(), equalTo(7L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(11));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(2));
diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
index e088e8569bf8a..cd6f596cfda05 100644
--- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
@@ -17,6 +17,7 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
@@ -310,7 +311,7 @@ public void testNestedSorting() throws Exception {
 
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopFieldDocs topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(7L));
+        assertThat(topDocs.totalHits.value(), equalTo(7L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(3));
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("a"));
@@ -332,7 +333,7 @@ public void testNestedSorting() throws Exception {
         );
         sort = new Sort(new SortField("field2", nestedComparatorSource, true));
         topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(7L));
+        assertThat(topDocs.totalHits.value(), equalTo(7L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(28));
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("o"));
@@ -358,7 +359,7 @@ public void testNestedSorting() throws Exception {
         query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None);
         sort = new Sort(new SortField("field2", nestedComparatorSource, true));
         topDocs = searcher.search(query, 5, sort);
-        assertThat(topDocs.totalHits.value, equalTo(6L));
+        assertThat(topDocs.totalHits.value(), equalTo(6L));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(23));
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("m"));
@@ -620,42 +621,43 @@ public void testMultiLevelNestedSorting() throws IOException {
         sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs")));
         QueryBuilder queryBuilder = new MatchAllQueryBuilder();
         TopFieldDocs topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-        assertThat(topFields.totalHits.value, equalTo(5L));
-        assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+        assertThat(topFields.totalHits.value(), equalTo(5L));
+        StoredFields storedFields = searcher.storedFields();
+        assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
         assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
-        assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("4"));
+        assertThat(storedFields.document(topFields.scoreDocs[1].doc).get("_id"), equalTo("4"));
         assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(87L));
-        assertThat(searcher.doc(topFields.scoreDocs[2].doc).get("_id"), equalTo("1"));
+        assertThat(storedFields.document(topFields.scoreDocs[2].doc).get("_id"), equalTo("1"));
         assertThat(((FieldDoc) topFields.scoreDocs[2]).fields[0], equalTo(234L));
-        assertThat(searcher.doc(topFields.scoreDocs[3].doc).get("_id"), equalTo("3"));
+        assertThat(storedFields.document(topFields.scoreDocs[3].doc).get("_id"), equalTo("3"));
         assertThat(((FieldDoc) topFields.scoreDocs[3]).fields[0], equalTo(976L));
-        assertThat(searcher.doc(topFields.scoreDocs[4].doc).get("_id"), equalTo("5"));
+        assertThat(storedFields.document(topFields.scoreDocs[4].doc).get("_id"), equalTo("5"));
         assertThat(((FieldDoc) topFields.scoreDocs[4]).fields[0], equalTo(Long.MAX_VALUE));
 
         // Specific genre
         {
             queryBuilder = new TermQueryBuilder("genre", "romance");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
 
             queryBuilder = new TermQueryBuilder("genre", "science fiction");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(234L));
 
             queryBuilder = new TermQueryBuilder("genre", "horror");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(976L));
 
             queryBuilder = new TermQueryBuilder("genre", "cooking");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
         }
 
@@ -664,16 +666,16 @@ public void testMultiLevelNestedSorting() throws IOException {
             sortBuilder.order(SortOrder.DESC);
             queryBuilder = new MatchAllQueryBuilder();
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(5L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(topFields.totalHits.value(), equalTo(5L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(976L));
-            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("1"));
+            assertThat(storedFields.document(topFields.scoreDocs[1].doc).get("_id"), equalTo("1"));
             assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(849L));
-            assertThat(searcher.doc(topFields.scoreDocs[2].doc).get("_id"), equalTo("4"));
+            assertThat(storedFields.document(topFields.scoreDocs[2].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[2]).fields[0], equalTo(180L));
-            assertThat(searcher.doc(topFields.scoreDocs[3].doc).get("_id"), equalTo("2"));
+            assertThat(storedFields.document(topFields.scoreDocs[3].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[3]).fields[0], equalTo(76L));
-            assertThat(searcher.doc(topFields.scoreDocs[4].doc).get("_id"), equalTo("5"));
+            assertThat(storedFields.document(topFields.scoreDocs[4].doc).get("_id"), equalTo("5"));
             assertThat(((FieldDoc) topFields.scoreDocs[4]).fields[0], equalTo(Long.MIN_VALUE));
         }
 
@@ -681,26 +683,26 @@ public void testMultiLevelNestedSorting() throws IOException {
         {
             queryBuilder = new TermQueryBuilder("genre", "romance");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
 
             queryBuilder = new TermQueryBuilder("genre", "science fiction");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(849L));
 
             queryBuilder = new TermQueryBuilder("genre", "horror");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(976L));
 
             queryBuilder = new TermQueryBuilder("genre", "cooking");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(180L));
         }
 
@@ -717,10 +719,10 @@ public void testMultiLevelNestedSorting() throws IOException {
                 searchExecutionContext,
                 searcher
             );
-            assertThat(topFields.totalHits.value, equalTo(2L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(topFields.totalHits.value(), equalTo(2L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
-            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("4"));
+            assertThat(storedFields.document(topFields.scoreDocs[1].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(87L));
 
             sortBuilder.order(SortOrder.DESC);
@@ -730,10 +732,10 @@ public void testMultiLevelNestedSorting() throws IOException {
                 searchExecutionContext,
                 searcher
             );
-            assertThat(topFields.totalHits.value, equalTo(2L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(topFields.totalHits.value(), equalTo(2L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
-            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
+            assertThat(storedFields.document(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(76L));
         }
 
@@ -755,10 +757,10 @@ public void testMultiLevelNestedSorting() throws IOException {
                 searchExecutionContext,
                 searcher
             );
-            assertThat(topFields.totalHits.value, equalTo(2L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(topFields.totalHits.value(), equalTo(2L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
-            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
+            assertThat(storedFields.document(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(Long.MAX_VALUE));
 
             sortBuilder.order(SortOrder.DESC);
@@ -768,10 +770,10 @@ public void testMultiLevelNestedSorting() throws IOException {
                 searchExecutionContext,
                 searcher
             );
-            assertThat(topFields.totalHits.value, equalTo(2L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(topFields.totalHits.value(), equalTo(2L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
-            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
+            assertThat(storedFields.document(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(Long.MIN_VALUE));
         }
 
@@ -785,26 +787,26 @@ public void testMultiLevelNestedSorting() throws IOException {
 
             queryBuilder = new TermQueryBuilder("genre", "romance");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
 
             queryBuilder = new TermQueryBuilder("genre", "science fiction");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(Long.MAX_VALUE));
 
             queryBuilder = new TermQueryBuilder("genre", "horror");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(Long.MAX_VALUE));
 
             queryBuilder = new TermQueryBuilder("genre", "cooking");
             topFields = search(queryBuilder, sortBuilder, searchExecutionContext, searcher);
-            assertThat(topFields.totalHits.value, equalTo(1L));
-            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(topFields.totalHits.value(), equalTo(1L));
+            assertThat(storedFields.document(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexReaderWrapperTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexReaderWrapperTests.java
index 8355f0156d0c3..7c7313d6b6516 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexReaderWrapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexReaderWrapperTests.java
@@ -52,7 +52,7 @@ public void testReaderCloseListenerIsCalled() throws IOException {
         writer.addDocument(doc);
         DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
         IndexSearcher searcher = newSearcher(open);
-        assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
+        assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value());
         final AtomicInteger closeCalls = new AtomicInteger(0);
         CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader(
             "field",
@@ -82,7 +82,7 @@ public void testReaderCloseListenerIsCalled() throws IOException {
             }
             outerCount.incrementAndGet();
         });
-        assertEquals(0, wrap.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
+        assertEquals(0, wrap.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value());
         wrap.close();
         assertFalse("wrapped reader is closed", wrap.getIndexReader().tryIncRef());
         assertEquals(sourceRefCount, open.getRefCount());
@@ -106,7 +106,7 @@ public void testIsCacheable() throws IOException {
         writer.addDocument(doc);
         DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
         IndexSearcher searcher = newSearcher(open);
-        assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
+        assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value());
         searcher.setSimilarity(iwc.getSimilarity());
         final AtomicInteger closeCalls = new AtomicInteger(0);
         CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader(
@@ -153,7 +153,7 @@ public void testAlwaysWrapWithFieldUsageTrackingDirectoryReader() throws IOExcep
         writer.addDocument(doc);
         DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
         IndexSearcher searcher = newSearcher(open);
-        assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
+        assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value());
         searcher.setSimilarity(iwc.getSimilarity());
         CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = directoryReader -> directoryReader;
         try (
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index f15506676dc39..d480f7bfc8d7f 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2793,9 +2793,9 @@ public void testReaderWrapperIsUsed() throws IOException {
         }
         try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
             TopDocs search = searcher.search(new TermQuery(new Term("foo", "bar")), 10);
-            assertEquals(search.totalHits.value, 1);
+            assertEquals(search.totalHits.value(), 1);
             search = searcher.search(new TermQuery(new Term("foobar", "bar")), 10);
-            assertEquals(search.totalHits.value, 1);
+            assertEquals(search.totalHits.value(), 1);
         }
         CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
         closeShards(shard);
@@ -2815,9 +2815,9 @@ public void testReaderWrapperIsUsed() throws IOException {
 
         try (Engine.Searcher searcher = newShard.acquireSearcher("test")) {
             TopDocs search = searcher.search(new TermQuery(new Term("foo", "bar")), 10);
-            assertEquals(search.totalHits.value, 0);
+            assertEquals(search.totalHits.value(), 0);
             search = searcher.search(new TermQuery(new Term("foobar", "bar")), 10);
-            assertEquals(search.totalHits.value, 1);
+            assertEquals(search.totalHits.value(), 1);
         }
         try (Engine.GetResult getResult = newShard.get(new Engine.Get(false, false, "1"))) {
             assertTrue(getResult.exists());
diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 6fa405c091da1..ccf0bbebcc354 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -437,9 +437,8 @@ public void testLotsOfThreads() throws Exception {
                         ) {
                             assertTrue("document not found", getResult.exists());
                             assertEquals(iteration, getResult.version());
-                            org.apache.lucene.document.Document document = getResult.docIdAndVersion().reader.document(
-                                getResult.docIdAndVersion().docId
-                            );
+                            org.apache.lucene.document.Document document = getResult.docIdAndVersion().reader.storedFields()
+                                .document(getResult.docIdAndVersion().docId);
                             assertThat(document.getValues("test"), arrayContaining(testFieldValue));
                         }
                     } catch (Exception t) {
diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java
index 70e5143552235..aa89f31757ef4 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreMode;
@@ -172,6 +173,7 @@ void assertSplit(Directory dir, IndexMetadata metadata, int targetShardId, boole
                     int doc;
                     int numActual = 0;
                     int lastDoc = 0;
+                    StoredFields storedFields = reader.storedFields();
                     while ((doc = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
                         lastDoc = doc;
                         while (shard_id.nextDoc() < doc) {
@@ -181,7 +183,7 @@ void assertSplit(Directory dir, IndexMetadata metadata, int targetShardId, boole
                         }
                         assertEquals(shard_id.docID(), doc);
                         long shardID = shard_id.nextValue();
-                        BytesRef id = reader.document(doc).getBinaryValue("_id");
+                        BytesRef id = storedFields.document(doc).getBinaryValue("_id");
                         String actualId = Uid.decodeId(id.bytes, id.offset, id.length);
                         assertNotEquals(ctx.reader() + " docID: " + doc + " actualID: " + actualId, shardID, targetShardId);
                     }
diff --git a/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java
index b05f287e628a3..fa5f713dfd672 100644
--- a/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java
+++ b/server/src/test/java/org/elasticsearch/index/similarity/ScriptedSimilarityTests.java
@@ -144,7 +144,7 @@ public double execute(
             3.2f
         );
         TopDocs topDocs = searcher.search(query, 1);
-        assertEquals(1, topDocs.totalHits.value);
+        assertEquals(1, topDocs.totalHits.value());
         assertTrue(called.get());
         assertEquals(42, topDocs.scoreDocs[0].score, 0);
         r.close();
@@ -236,7 +236,7 @@ public double execute(
         searcher.setSimilarity(sim);
         Query query = new BoostQuery(new TermQuery(new Term("f", "foo")), 3.2f);
         TopDocs topDocs = searcher.search(query, 1);
-        assertEquals(1, topDocs.totalHits.value);
+        assertEquals(1, topDocs.totalHits.value());
         assertTrue(initCalled.get());
         assertTrue(called.get());
         assertEquals(42, topDocs.scoreDocs[0].score, 0);
diff --git a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java
index 394ce35c6b493..b0a14515f2fbc 100644
--- a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java
+++ b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java
@@ -33,10 +33,16 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiPredicate;
 
 public class FsDirectoryFactoryTests extends ESTestCase {
 
+    final PreLoadExposingFsDirectoryFactory fsDirectoryFactory = new PreLoadExposingFsDirectoryFactory();
+
     public void testPreload() throws IOException {
         doTestPreload();
         doTestPreload("nvd", "dvd", "tim");
@@ -60,10 +66,11 @@ public void testPreload() throws IOException {
             assertTrue(FsDirectoryFactory.HybridDirectory.useDelegate("foo.tmp", newIOContext(random())));
             assertTrue(FsDirectoryFactory.HybridDirectory.useDelegate("foo.fdt__0.tmp", newIOContext(random())));
             MMapDirectory delegate = hybridDirectory.getDelegate();
-            assertThat(delegate, Matchers.instanceOf(FsDirectoryFactory.PreLoadMMapDirectory.class));
-            FsDirectoryFactory.PreLoadMMapDirectory preLoadMMapDirectory = (FsDirectoryFactory.PreLoadMMapDirectory) delegate;
-            assertTrue(preLoadMMapDirectory.useDelegate("foo.dvd"));
-            assertTrue(preLoadMMapDirectory.useDelegate("foo.tmp"));
+            assertThat(delegate, Matchers.instanceOf(MMapDirectory.class));
+            var func = fsDirectoryFactory.preLoadFuncMap.get(delegate);
+            assertTrue(func.test("foo.dvd", newIOContext(random())));
+            assertTrue(func.test("foo.tmp", newIOContext(random())));
+            fsDirectoryFactory.preLoadFuncMap.clear();
         }
     }
 
@@ -72,7 +79,21 @@ private Directory newDirectory(Settings settings) throws IOException {
         Path tempDir = createTempDir().resolve(idxSettings.getUUID()).resolve("0");
         Files.createDirectories(tempDir);
         ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(idxSettings.getIndex(), 0));
-        return new FsDirectoryFactory().newDirectory(idxSettings, path);
+        return fsDirectoryFactory.newDirectory(idxSettings, path);
+    }
+
+    static class PreLoadExposingFsDirectoryFactory extends FsDirectoryFactory {
+
+        // expose for testing
+        final Map<MMapDirectory, BiPredicate<String, IOContext>> preLoadFuncMap = new HashMap<>();
+
+        @Override
+        public MMapDirectory setPreload(MMapDirectory mMapDirectory, Set<String> preLoadExtensions) {
+            var preLoadFunc = FsDirectoryFactory.getPreloadFunc(preLoadExtensions);
+            mMapDirectory.setPreload(preLoadFunc);
+            preLoadFuncMap.put(mMapDirectory, preLoadFunc);
+            return mMapDirectory;
+        }
     }
 
     private void doTestPreload(String... preload) throws IOException {
@@ -85,26 +106,21 @@ private void doTestPreload(String... preload) throws IOException {
             assertSame(dir, directory); // prevent warnings
             assertFalse(directory instanceof SleepingLockWrapper);
             var mmapDirectory = FilterDirectory.unwrap(directory);
+            assertTrue(directory.toString(), mmapDirectory instanceof MMapDirectory);
             if (preload.length == 0) {
-                assertTrue(directory.toString(), mmapDirectory instanceof MMapDirectory);
-                assertFalse(((MMapDirectory) mmapDirectory).getPreload());
+                assertEquals(fsDirectoryFactory.preLoadFuncMap.get(mmapDirectory), MMapDirectory.NO_FILES);
             } else if (Arrays.asList(preload).contains("*")) {
-                assertTrue(directory.toString(), mmapDirectory instanceof MMapDirectory);
-                assertTrue(((MMapDirectory) mmapDirectory).getPreload());
+                assertEquals(fsDirectoryFactory.preLoadFuncMap.get(mmapDirectory), MMapDirectory.ALL_FILES);
             } else {
-                assertTrue(directory.toString(), mmapDirectory instanceof FsDirectoryFactory.PreLoadMMapDirectory);
-                FsDirectoryFactory.PreLoadMMapDirectory preLoadMMapDirectory = (FsDirectoryFactory.PreLoadMMapDirectory) mmapDirectory;
+                var func = fsDirectoryFactory.preLoadFuncMap.get(mmapDirectory);
+                assertNotEquals(fsDirectoryFactory.preLoadFuncMap.get(mmapDirectory), MMapDirectory.ALL_FILES);
+                assertNotEquals(fsDirectoryFactory.preLoadFuncMap.get(mmapDirectory), MMapDirectory.NO_FILES);
                 for (String ext : preload) {
-                    assertTrue("ext: " + ext, preLoadMMapDirectory.useDelegate("foo." + ext));
-                    assertTrue("ext: " + ext, preLoadMMapDirectory.getDelegate().getPreload());
+                    assertTrue("ext: " + ext, func.test("foo." + ext, newIOContext(random())));
                 }
-                assertFalse(preLoadMMapDirectory.useDelegate("XXX"));
-                assertFalse(preLoadMMapDirectory.getPreload());
-                preLoadMMapDirectory.close();
-                expectThrows(
-                    AlreadyClosedException.class,
-                    () -> preLoadMMapDirectory.getDelegate().openInput("foo.tmp", IOContext.DEFAULT)
-                );
+                assertFalse(func.test("XXX", newIOContext(random())));
+                mmapDirectory.close();
+                expectThrows(AlreadyClosedException.class, () -> mmapDirectory.openInput("foo.tmp", IOContext.DEFAULT));
             }
         }
         expectThrows(
@@ -148,7 +164,7 @@ private void doTestStoreDirectory(Path tempDir, String typeSettingValue, IndexMo
                     );
                     break;
                 case FS:
-                    if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
+                    if (Constants.JRE_IS_64BIT) {
                         assertTrue(FsDirectoryFactory.isHybridFs(directory));
                     } else {
                         assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
index ebffd54a742ce..4f73672471942 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
@@ -70,8 +70,9 @@ public String toString(String field) {
         public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
             return new ConstantScoreWeight(this, boost) {
                 @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
-                    return new ConstantScoreScorer(this, score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc()));
+                public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                    Scorer scorer = new ConstantScoreScorer(score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc()));
+                    return new DefaultScorerSupplier(scorer);
                 }
 
                 @Override
@@ -348,16 +349,22 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
             return weight.explain(context, doc);
         }
 
-        @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            scorerCalled = true;
-            return weight.scorer(context);
-        }
-
         @Override
         public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
             scorerSupplierCalled = true;
-            return weight.scorerSupplier(context);
+            ScorerSupplier inScorerSupplier = weight.scorerSupplier(context);
+            return new ScorerSupplier() {
+                @Override
+                public Scorer get(long leadCost) throws IOException {
+                    scorerCalled = true;
+                    return inScorerSupplier.get(leadCost);
+                }
+
+                @Override
+                public long cost() {
+                    return inScorerSupplier.cost();
+                }
+            };
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
index dfd71bba0208c..773c660caa1c6 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
@@ -421,8 +421,8 @@ public BytesReference get() {
             try (BytesStreamOutput out = new BytesStreamOutput()) {
                 IndexSearcher searcher = newSearcher(reader);
                 TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1);
-                assertEquals(1, topDocs.totalHits.value);
-                Document document = reader.document(topDocs.scoreDocs[0].doc);
+                assertEquals(1, topDocs.totalHits.value());
+                Document document = reader.storedFields().document(topDocs.scoreDocs[0].doc);
                 out.writeString(document.get("value"));
                 loadedFromCache = false;
                 return out.bytes();
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
index 3adaf398624de..d83fdbd5dd46b 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.internal.Client;
@@ -48,6 +49,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
@@ -424,7 +426,7 @@ public void testDelete() {
 
     public void testValidateNoIngestInfo() throws Exception {
         IngestService ingestService = createWithProcessors();
-        PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """
+        PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {"processors": [{"set" : {"field": "_field", "value": "_value"}}]}""");
 
         var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
@@ -965,7 +967,7 @@ public void testGetPipelines() {
 
     public void testValidateProcessorTypeOnAllNodes() throws Exception {
         IngestService ingestService = createWithProcessors();
-        PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """
+        PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {
               "processors": [
                 {
@@ -1009,7 +1011,7 @@ public void testValidateConfigurationExceptions() {
             // ordinary validation issues happen at processor construction time
             throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason");
         }));
-        PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """
+        PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {
               "processors": [
                 {
@@ -1043,7 +1045,7 @@ public void extraValidation() throws Exception {
                 }
             };
         }));
-        PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """
+        PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {
               "processors": [
                 {
@@ -1067,6 +1069,32 @@ public void extraValidation() throws Exception {
         assertEquals("fail_extra_validation", e.getMetadata("es.processor_type").get(0));
     }
 
+    public void testValidatePipelineName() throws Exception {
+        IngestService ingestService = createWithProcessors();
+        for (Character badChar : List.of('\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',')) {
+            PutPipelineRequest putRequest = new PutPipelineRequest(
+                TimeValue.timeValueSeconds(10),
+                AcknowledgedRequest.DEFAULT_ACK_TIMEOUT,
+                "_id",
+                new BytesArray("""
+                    {"description":"test processor","processors":[{"set":{"field":"_field","value":"_value"}}]}"""),
+                XContentType.JSON
+            );
+            var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
+            DiscoveryNode node1 = DiscoveryNodeUtils.create("_node_id1", buildNewFakeTransportAddress(), Map.of(), Set.of());
+            Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
+            ingestInfos.put(node1, new IngestInfo(List.of(new ProcessorInfo("set"))));
+            final String name = randomAlphaOfLength(5) + badChar + randomAlphaOfLength(5);
+            ingestService.validatePipeline(ingestInfos, name, pipelineConfig);
+            assertCriticalWarnings(
+                "Pipeline name ["
+                    + name
+                    + "] will be disallowed in a future version for the following reason: must not contain the following characters"
+                    + " [' ','\"','*',',','/','<','>','?','\\','|']"
+            );
+        }
+    }
+
     public void testExecuteIndexPipelineExistsButFailedParsing() {
         IngestService ingestService = createWithProcessors(
             Map.of("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") {
diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java
index 3b3f5bdc747b5..94b3607bd7608 100644
--- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java
@@ -65,7 +65,7 @@ public void testGetPipeline() {
         ingestService.innerUpdatePipelines(ingestMetadata);
         {
             // First we make sure that if there are no substitutions that we get our original pipeline back:
-            SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null, null);
+            SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
             SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest);
             Pipeline pipeline = simulateIngestService.getPipeline("pipeline1");
             assertThat(pipeline.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1"))));
@@ -83,7 +83,7 @@ public void testGetPipeline() {
             );
             pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap()))));
 
-            SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null);
+            SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, Map.of(), Map.of(), Map.of());
             SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest);
             Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1");
             assertThat(
@@ -103,7 +103,7 @@ public void testGetPipeline() {
              */
             Map<String, Map<String, Object>> pipelineSubstitutions = new HashMap<>();
             pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap()))));
-            SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null);
+            SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, Map.of(), Map.of(), Map.of());
             SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest);
             Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1");
             assertThat(pipeline1.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1"))));
diff --git a/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorSearchAfterTests.java b/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorSearchAfterTests.java
index baa35101c1c87..1c1a9a645b99b 100644
--- a/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorSearchAfterTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorSearchAfterTests.java
@@ -114,11 +114,11 @@ private <T extends Comparable<T>> void assertSearchCollapse(
         TopFieldDocs topDocs = searcher.search(query, topFieldCollectorManager);
         TopFieldGroups collapseTopFieldDocs = collapsingCollector.getTopGroups(0);
         assertEquals(sortField.getField(), collapseTopFieldDocs.field);
-        assertEquals(totalHits, collapseTopFieldDocs.totalHits.value);
+        assertEquals(totalHits, collapseTopFieldDocs.totalHits.value());
         assertEquals(expectedNumGroups, collapseTopFieldDocs.scoreDocs.length);
 
-        assertEquals(TotalHits.Relation.EQUAL_TO, collapseTopFieldDocs.totalHits.relation);
-        assertEquals(totalHits, topDocs.totalHits.value);
+        assertEquals(TotalHits.Relation.EQUAL_TO, collapseTopFieldDocs.totalHits.relation());
+        assertEquals(totalHits, topDocs.totalHits.value());
 
         Object currentValue = null;
         int topDocsIndex = 0;
diff --git a/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorTests.java b/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorTests.java
index 90adb2d0ffcce..30c68fe708c83 100644
--- a/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollectorTests.java
@@ -49,11 +49,11 @@
 
 public class SinglePassGroupingCollectorTests extends ESTestCase {
     private static class SegmentSearcher extends IndexSearcher {
-        private final List<LeafReaderContext> ctx;
+        private final LeafReaderContextPartition[] ctx;
 
         SegmentSearcher(LeafReaderContext ctx, IndexReaderContext parent) {
             super(parent);
-            this.ctx = Collections.singletonList(ctx);
+            this.ctx = new LeafReaderContextPartition[] { IndexSearcher.LeafReaderContextPartition.createForEntireSegment(ctx) };
         }
 
         public void search(Weight weight, Collector collector) throws IOException {
@@ -62,7 +62,7 @@ public void search(Weight weight, Collector collector) throws IOException {
 
         @Override
         public String toString() {
-            return "ShardSearcher(" + ctx.get(0) + ")";
+            return "ShardSearcher(" + ctx[0] + ")";
         }
     }
 
@@ -140,10 +140,10 @@ private <T extends Comparable<T>> void assertSearchCollapse(
         TopFieldGroups collapseTopFieldDocs = collapsingCollector.getTopGroups(0);
         assertEquals(collapseField.getField(), collapseTopFieldDocs.field);
         assertEquals(expectedNumGroups, collapseTopFieldDocs.scoreDocs.length);
-        assertEquals(totalHits, collapseTopFieldDocs.totalHits.value);
-        assertEquals(TotalHits.Relation.EQUAL_TO, collapseTopFieldDocs.totalHits.relation);
+        assertEquals(totalHits, collapseTopFieldDocs.totalHits.value());
+        assertEquals(TotalHits.Relation.EQUAL_TO, collapseTopFieldDocs.totalHits.relation());
         assertEquals(totalHits, topDocs.scoreDocs.length);
-        assertEquals(totalHits, topDocs.totalHits.value);
+        assertEquals(totalHits, topDocs.totalHits.value());
 
         Set<Object> seen = new HashSet<>();
         // collapse field is the last sort
diff --git a/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java
index 71a2703555318..076cd0af1bf26 100644
--- a/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/queries/BlendedTermQueryTests.java
@@ -88,7 +88,7 @@ public void testDismaxQuery() throws IOException {
             query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "generator"), 0.1f), BooleanClause.Occur.SHOULD);
             TopDocs search = searcher.search(query.build(), 10);
             ScoreDoc[] scoreDocs = search.scoreDocs;
-            assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue());
+            assertEquals(Integer.toString(0), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue());
         }
         {
             BooleanQuery.Builder query = new BooleanQuery.Builder();
@@ -110,7 +110,7 @@ public void testDismaxQuery() throws IOException {
             query.add(gen, BooleanClause.Occur.SHOULD);
             TopDocs search = searcher.search(query.build(), 4);
             ScoreDoc[] scoreDocs = search.scoreDocs;
-            assertEquals(Integer.toString(1), reader.document(scoreDocs[0].doc).getField("id").stringValue());
+            assertEquals(Integer.toString(1), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue());
 
         }
         {
@@ -120,8 +120,8 @@ public void testDismaxQuery() throws IOException {
             Query rewrite = searcher.rewrite(query);
             assertThat(rewrite, instanceOf(BooleanQuery.class));
             for (BooleanClause clause : (BooleanQuery) rewrite) {
-                assertThat(clause.getQuery(), instanceOf(TermQuery.class));
-                TermQuery termQuery = (TermQuery) clause.getQuery();
+                assertThat(clause.query(), instanceOf(TermQuery.class));
+                TermQuery termQuery = (TermQuery) clause.query();
                 TermStates termStates = termQuery.getTermStates();
                 if (termQuery.getTerm().field().equals("unknown_field")) {
                     assertThat(termStates.docFreq(), equalTo(0));
@@ -131,7 +131,7 @@ public void testDismaxQuery() throws IOException {
                     assertThat(termStates.totalTermFreq(), greaterThan(0L));
                 }
             }
-            assertThat(searcher.search(query, 10).totalHits.value, equalTo((long) iters + username.length));
+            assertThat(searcher.search(query, 10).totalHits.value(), equalTo((long) iters + username.length));
         }
         {
             // test with an unknown field and an unknown term
@@ -140,13 +140,13 @@ public void testDismaxQuery() throws IOException {
             Query rewrite = searcher.rewrite(query);
             assertThat(rewrite, instanceOf(BooleanQuery.class));
             for (BooleanClause clause : (BooleanQuery) rewrite) {
-                assertThat(clause.getQuery(), instanceOf(TermQuery.class));
-                TermQuery termQuery = (TermQuery) clause.getQuery();
+                assertThat(clause.query(), instanceOf(TermQuery.class));
+                TermQuery termQuery = (TermQuery) clause.query();
                 TermStates termStates = termQuery.getTermStates();
                 assertThat(termStates.docFreq(), equalTo(0));
                 assertThat(termStates.totalTermFreq(), equalTo(0L));
             }
-            assertThat(searcher.search(query, 10).totalHits.value, equalTo(0L));
+            assertThat(searcher.search(query, 10).totalHits.value(), equalTo(0L));
         }
         {
             // test with an unknown field and a term that is present in only one field
@@ -155,8 +155,8 @@ public void testDismaxQuery() throws IOException {
             Query rewrite = searcher.rewrite(query);
             assertThat(rewrite, instanceOf(BooleanQuery.class));
             for (BooleanClause clause : (BooleanQuery) rewrite) {
-                assertThat(clause.getQuery(), instanceOf(TermQuery.class));
-                TermQuery termQuery = (TermQuery) clause.getQuery();
+                assertThat(clause.query(), instanceOf(TermQuery.class));
+                TermQuery termQuery = (TermQuery) clause.query();
                 TermStates termStates = termQuery.getTermStates();
                 if (termQuery.getTerm().field().equals("username")) {
                     assertThat(termStates.docFreq(), equalTo(1));
@@ -166,7 +166,7 @@ public void testDismaxQuery() throws IOException {
                     assertThat(termStates.totalTermFreq(), equalTo(0L));
                 }
             }
-            assertThat(searcher.search(query, 10).totalHits.value, equalTo(1L));
+            assertThat(searcher.search(query, 10).totalHits.value(), equalTo(1L));
         }
         reader.close();
         w.close();
@@ -250,7 +250,7 @@ public void testMinTTF() throws IOException {
             Query query = BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "foo"), 0.1f);
             TopDocs search = searcher.search(query, 10);
             ScoreDoc[] scoreDocs = search.scoreDocs;
-            assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue());
+            assertEquals(Integer.toString(0), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue());
         }
         reader.close();
         w.close();
@@ -298,7 +298,7 @@ public void testRandomFields() throws IOException {
             String[] fieldNames = fields.keySet().toArray(new String[0]);
             Query query = BlendedTermQuery.dismaxBlendedQuery(toTerms(fieldNames, "foo"), 0.1f);
             TopDocs search = searcher.search(query, 10);
-            assertTrue(search.totalHits.value > 0);
+            assertTrue(search.totalHits.value() > 0);
             assertTrue(search.scoreDocs.length > 0);
         }
         reader.close();
@@ -332,7 +332,7 @@ public void testMissingFields() throws IOException {
         Query query = BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "foo"), 0.1f);
         TopDocs search = searcher.search(query, 10);
         ScoreDoc[] scoreDocs = search.scoreDocs;
-        assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue());
+        assertEquals(Integer.toString(0), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue());
 
         reader.close();
         w.close();
diff --git a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
index f5266568e6fdf..126641037fde7 100644
--- a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
@@ -141,7 +141,7 @@ private void assertHighlightOneDoc(
                 IndexSearcher searcher = newSearcher(reader);
                 iw.close();
                 TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 1, Sort.INDEXORDER);
-                assertThat(topDocs.totalHits.value, equalTo(1L));
+                assertThat(topDocs.totalHits.value(), equalTo(1L));
                 String rawValue = Strings.arrayToDelimitedString(inputs, String.valueOf(MULTIVAL_SEP_CHAR));
                 UnifiedHighlighter.Builder builder = UnifiedHighlighter.builder(searcher, analyzer);
                 builder.withBreakIterator(() -> breakIterator);
diff --git a/server/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java b/server/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java
index 12fa776dd7efd..28976d803ff53 100644
--- a/server/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java
@@ -13,17 +13,22 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
 
 public class JvmStatsTests extends ESTestCase {
-    public void testJvmStats() throws IOException {
+    public void testJvmStats() {
         JvmStats stats = JvmStats.jvmStats();
         assertNotNull(stats);
         assertNotNull(stats.getUptime());
@@ -40,6 +45,17 @@ public void testJvmStats() throws IOException {
         assertNotNull(mem.getHeapUsedPercent());
         assertThat(mem.getHeapUsedPercent(), anyOf(equalTo((short) -1), greaterThanOrEqualTo((short) 0)));
 
+        // Memory pools
+        Map<String, JvmStats.MemoryPool> memoryPools = StreamSupport.stream(stats.getMem().spliterator(), false)
+            .collect(Collectors.toMap(JvmStats.MemoryPool::getName, Function.identity()));
+        assertThat(memoryPools, hasKey(GcNames.YOUNG));
+        assertThat(memoryPools, hasKey(GcNames.OLD));
+        assertThat(memoryPools, hasKey("Metaspace"));
+        assertThat(memoryPools.keySet(), hasSize(greaterThan(3)));
+        for (JvmStats.MemoryPool memoryPool : memoryPools.values()) {
+            assertThat(memoryPool.getUsed().getBytes(), greaterThan(0L));
+        }
+
         // Threads
         JvmStats.Threads threads = stats.getThreads();
         assertNotNull(threads);
diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
index aa6a9667ce39e..8af36e2f9677e 100644
--- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.reservedstate.service;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
@@ -26,6 +25,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
 import org.elasticsearch.tasks.TaskManager;
@@ -54,6 +54,7 @@
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
@@ -148,9 +149,9 @@ public void testOperatorDirName() {
     @SuppressWarnings("unchecked")
     public void testInitialFileError() throws Exception {
         doAnswer((Answer<Void>) invocation -> {
-            ((Consumer<Exception>) invocation.getArgument(2)).accept(new IllegalStateException("Some exception"));
+            ((Consumer<Exception>) invocation.getArgument(3)).accept(new IllegalStateException("Some exception"));
             return null;
-        }).when(controller).process(any(), any(XContentParser.class), any());
+        }).when(controller).process(any(), any(XContentParser.class), eq(randomFrom(ReservedStateVersionCheck.values())), any());
 
         AtomicBoolean settingsChanged = new AtomicBoolean(false);
         CountDownLatch latch = new CountDownLatch(1);
@@ -163,7 +164,7 @@ public void testInitialFileError() throws Exception {
             } finally {
                 latch.countDown();
             }
-        }).when(fileSettingsService).processFileChanges();
+        }).when(fileSettingsService).processFileOnServiceStart();
 
         Files.createDirectories(fileSettingsService.watchedFileDir());
         // contents of the JSON don't matter, we just need a file to exist
@@ -175,7 +176,8 @@ public void testInitialFileError() throws Exception {
         // wait until the watcher thread has started, and it has discovered the file
         assertTrue(latch.await(20, TimeUnit.SECONDS));
 
-        verify(fileSettingsService, times(1)).processFileChanges();
+        verify(fileSettingsService, times(1)).processFileOnServiceStart();
+        verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any());
         // assert we never notified any listeners of successful application of file based settings
         assertFalse(settingsChanged.get());
     }
@@ -184,9 +186,9 @@ public void testInitialFileError() throws Exception {
     public void testInitialFileWorks() throws Exception {
         // Let's check that if we didn't throw an error that everything works
         doAnswer((Answer<Void>) invocation -> {
-            ((Consumer<Exception>) invocation.getArgument(2)).accept(null);
+            ((Consumer<Exception>) invocation.getArgument(3)).accept(null);
             return null;
-        }).when(controller).process(any(), any(XContentParser.class), any());
+        }).when(controller).process(any(), any(XContentParser.class), any(), any());
 
         CountDownLatch latch = new CountDownLatch(1);
 
@@ -196,13 +198,67 @@ public void testInitialFileWorks() throws Exception {
         // contents of the JSON don't matter, we just need a file to exist
         writeTestFile(fileSettingsService.watchedFile(), "{}");
 
+        doAnswer((Answer<?>) invocation -> {
+            try {
+                return invocation.callRealMethod();
+            } finally {
+                latch.countDown();
+            }
+        }).when(fileSettingsService).processFileOnServiceStart();
+
         fileSettingsService.start();
         fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
 
         // wait for listener to be called
         assertTrue(latch.await(20, TimeUnit.SECONDS));
 
+        verify(fileSettingsService, times(1)).processFileOnServiceStart();
+        verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testProcessFileChanges() throws Exception {
+        doAnswer((Answer<Void>) invocation -> {
+            ((Consumer<Exception>) invocation.getArgument(3)).accept(null);
+            return null;
+        }).when(controller).process(any(), any(XContentParser.class), any(), any());
+
+        // we get three events: initial clusterChanged event, first write, second write
+        CountDownLatch latch = new CountDownLatch(3);
+
+        fileSettingsService.addFileChangedListener(latch::countDown);
+
+        Files.createDirectories(fileSettingsService.watchedFileDir());
+        // contents of the JSON don't matter, we just need a file to exist
+        writeTestFile(fileSettingsService.watchedFile(), "{}");
+
+        doAnswer((Answer<?>) invocation -> {
+            try {
+                return invocation.callRealMethod();
+            } finally {
+                latch.countDown();
+            }
+        }).when(fileSettingsService).processFileOnServiceStart();
+        doAnswer((Answer<?>) invocation -> {
+            try {
+                return invocation.callRealMethod();
+            } finally {
+                latch.countDown();
+            }
+        }).when(fileSettingsService).processFileChanges();
+
+        fileSettingsService.start();
+        fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
+        // second file change; contents still don't matter
+        overwriteTestFile(fileSettingsService.watchedFile(), "{}");
+
+        // wait for listener to be called (once for initial processing, once for subsequent update)
+        assertTrue(latch.await(20, TimeUnit.SECONDS));
+
+        verify(fileSettingsService, times(1)).processFileOnServiceStart();
+        verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any());
         verify(fileSettingsService, times(1)).processFileChanges();
+        verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_VERSION_ONLY), any());
     }
 
     @SuppressWarnings("unchecked")
@@ -221,7 +277,7 @@ public void testStopWorksInMiddleOfProcessing() throws Exception {
                     throw new RuntimeException(e);
                 }
             }).start();
-            return new ReservedStateChunk(Map.of(), new ReservedStateVersion(1L, Version.CURRENT));
+            return new ReservedStateChunk(Map.of(), new ReservedStateVersion(1L, BuildVersion.current()));
         }).when(controller).parse(any(String.class), any());
 
         doAnswer((Answer<Void>) invocation -> {
@@ -299,6 +355,12 @@ public void testHandleSnapshotRestoreResetsMetadata() throws Exception {
     private void writeTestFile(Path path, String contents) throws IOException {
         Path tempFilePath = createTempFile();
         Files.writeString(tempFilePath, contents);
-        Files.move(tempFilePath, path, StandardCopyOption.ATOMIC_MOVE);
+        Files.move(tempFilePath, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+    }
+
+    private void overwriteTestFile(Path path, String contents) throws IOException {
+        Path tempFilePath = createTempFile();
+        Files.writeString(tempFilePath, contents);
+        Files.move(tempFilePath, path, StandardCopyOption.REPLACE_EXISTING);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java
index 217b82d7729ae..5c7dd6cb346b9 100644
--- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.reservedstate.service;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -26,6 +25,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
@@ -167,7 +167,12 @@ public void testOperatorController() throws IOException {
         AtomicReference<Exception> x = new AtomicReference<>();
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) {
-            controller.process("operator", parser, x::set);
+            controller.process(
+                "operator",
+                parser,
+                randomFrom(ReservedStateVersionCheck.HIGHER_VERSION_ONLY, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION),
+                x::set
+            );
 
             assertThat(x.get(), instanceOf(IllegalStateException.class));
             assertThat(x.get().getMessage(), containsString("Error processing state change request for operator"));
@@ -197,7 +202,12 @@ public void testOperatorController() throws IOException {
             """;
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) {
-            controller.process("operator", parser, Assert::assertNull);
+            controller.process(
+                "operator",
+                parser,
+                randomFrom(ReservedStateVersionCheck.HIGHER_VERSION_ONLY, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION),
+                Assert::assertNull
+            );
         }
     }
 
@@ -236,7 +246,15 @@ public void testUpdateStateTasks() throws Exception {
         AtomicBoolean successCalled = new AtomicBoolean(false);
 
         ReservedStateUpdateTask task = spy(
-            new ReservedStateUpdateTask("test", null, Map.of(), Set.of(), errorState -> {}, ActionListener.noop())
+            new ReservedStateUpdateTask(
+                "test",
+                null,
+                ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+                Map.of(),
+                Set.of(),
+                errorState -> {},
+                ActionListener.noop()
+            )
         );
 
         doReturn(state).when(task).execute(any());
@@ -275,7 +293,13 @@ public void testUpdateErrorState() {
 
         ReservedClusterStateService service = new ReservedClusterStateService(clusterService, mock(RerouteService.class), List.of());
 
-        ErrorState error = new ErrorState("namespace", 2L, List.of("error"), ReservedStateErrorMetadata.ErrorKind.TRANSIENT);
+        ErrorState error = new ErrorState(
+            "namespace",
+            2L,
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+            List.of("error"),
+            ReservedStateErrorMetadata.ErrorKind.TRANSIENT
+        );
         service.updateErrorState(error);
 
         assertThat(updateTask.getValue(), notNullValue());
@@ -296,7 +320,13 @@ public void testUpdateErrorState() {
 
         // it should not update if the error version is less than the current version
         when(clusterService.state()).thenReturn(updatedState);
-        ErrorState oldError = new ErrorState("namespace", 1L, List.of("old error"), ReservedStateErrorMetadata.ErrorKind.TRANSIENT);
+        ErrorState oldError = new ErrorState(
+            "namespace",
+            1L,
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+            List.of("old error"),
+            ReservedStateErrorMetadata.ErrorKind.TRANSIENT
+        );
         service.updateErrorState(oldError);
         verifyNoMoreInteractions(errorQueue);
     }
@@ -308,7 +338,13 @@ public void testErrorStateTask() throws Exception {
 
         ReservedStateErrorTask task = spy(
             new ReservedStateErrorTask(
-                new ErrorState("test", 1L, List.of("some parse error", "some io error"), ReservedStateErrorMetadata.ErrorKind.PARSING),
+                new ErrorState(
+                    "test",
+                    1L,
+                    ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+                    List.of("some parse error", "some io error"),
+                    ReservedStateErrorMetadata.ErrorKind.PARSING
+                ),
                 ActionListener.running(() -> listenerCompleted.set(true))
             )
         );
@@ -353,12 +389,14 @@ public TransformState transform(Object source, TransformState prevState) throws
         Metadata metadata = Metadata.builder().put(operatorMetadata).build();
         ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build();
 
-        assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 2L));
-        assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 1L));
-        assertTrue(ReservedStateErrorTask.isNewError(operatorMetadata, 3L));
-        assertTrue(ReservedStateErrorTask.isNewError(null, 1L));
+        assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 2L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY));
+        assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 1L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY));
+        assertTrue(ReservedStateErrorTask.isNewError(operatorMetadata, 2L, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION));
+        assertTrue(ReservedStateErrorTask.isNewError(operatorMetadata, 3L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY));
+        assertTrue(ReservedStateErrorTask.isNewError(null, 1L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY));
+        assertTrue(ReservedStateErrorTask.isNewError(null, 1L, ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION));
 
-        var chunk = new ReservedStateChunk(Map.of("one", "two", "maker", "three"), new ReservedStateVersion(2L, Version.CURRENT));
+        var chunk = new ReservedStateChunk(Map.of("one", "two", "maker", "three"), new ReservedStateVersion(2L, BuildVersion.current()));
         var orderedHandlers = List.of(exceptionThrower.name(), newStateMaker.name());
 
         // We submit a task with two handler, one will cause an exception, the other will create a new state.
@@ -367,9 +405,10 @@ public TransformState transform(Object source, TransformState prevState) throws
         ReservedStateUpdateTask task = new ReservedStateUpdateTask(
             "namespace_one",
             chunk,
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
             Map.of(exceptionThrower.name(), exceptionThrower, newStateMaker.name(), newStateMaker),
             orderedHandlers,
-            errorState -> assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, errorState.version())),
+            errorState -> assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, errorState.version(), errorState.versionCheck())),
             ActionListener.noop()
         );
 
@@ -414,9 +453,21 @@ public void testCheckMetadataVersion() {
         ReservedStateMetadata operatorMetadata = ReservedStateMetadata.builder("test").version(123L).build();
 
         ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(Metadata.builder().put(operatorMetadata)).build();
+
         ReservedStateUpdateTask task = new ReservedStateUpdateTask(
             "test",
-            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.CURRENT)),
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersion.current())),
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+            Map.of(),
+            List.of(),
+            e -> {},
+            ActionListener.noop()
+        );
+        assertThat("Cluster state should be modified", task.execute(state), not(sameInstance(state)));
+        task = new ReservedStateUpdateTask(
+            "test",
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersion.current())),
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
             Map.of(),
             List.of(),
             e -> {},
@@ -426,17 +477,60 @@ public void testCheckMetadataVersion() {
 
         task = new ReservedStateUpdateTask(
             "test",
-            new ReservedStateChunk(Map.of(), new ReservedStateVersion(123L, Version.CURRENT)),
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(123L, BuildVersion.current())),
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
             Map.of(),
             List.of(),
             e -> {},
             ActionListener.noop()
         );
         assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state));
+        task = new ReservedStateUpdateTask(
+            "test",
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(123L, BuildVersion.current())),
+            ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION,
+            Map.of(),
+            List.of(),
+            e -> {},
+            ActionListener.noop()
+        );
+        assertThat("Cluster state should be modified", task.execute(state), not(sameInstance(state)));
 
         task = new ReservedStateUpdateTask(
             "test",
-            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.fromId(Version.CURRENT.id + 1))),
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(122L, BuildVersion.current())),
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+            Map.of(),
+            List.of(),
+            e -> {},
+            ActionListener.noop()
+        );
+        assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state));
+        task = new ReservedStateUpdateTask(
+            "test",
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(122L, BuildVersion.current())),
+            ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION,
+            Map.of(),
+            List.of(),
+            e -> {},
+            ActionListener.noop()
+        );
+        assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state));
+
+        task = new ReservedStateUpdateTask(
+            "test",
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersion.fromVersionId(BuildVersion.current().id() + 1))),
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+            Map.of(),
+            List.of(),
+            e -> {},
+            ActionListener.noop()
+        );
+        assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state));
+        task = new ReservedStateUpdateTask(
+            "test",
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersion.fromVersionId(BuildVersion.current().id() + 1))),
+            ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION,
             Map.of(),
             List.of(),
             e -> {},
@@ -530,11 +624,11 @@ public void testCheckAndReportError() {
 
         final var controller = spy(new ReservedClusterStateService(clusterService, mock(RerouteService.class), List.of()));
 
-        assertNull(controller.checkAndReportError("test", List.of(), null));
+        assertNull(controller.checkAndReportError("test", List.of(), null, ReservedStateVersionCheck.HIGHER_VERSION_ONLY));
         verify(controller, times(0)).updateErrorState(any());
 
-        var version = new ReservedStateVersion(2L, Version.CURRENT);
-        var error = controller.checkAndReportError("test", List.of("test error"), version);
+        var version = new ReservedStateVersion(2L, BuildVersion.current());
+        var error = controller.checkAndReportError("test", List.of("test error"), version, ReservedStateVersionCheck.HIGHER_VERSION_ONLY);
         assertThat(error, instanceOf(IllegalStateException.class));
         assertThat(error.getMessage(), is("Error processing state change request for test, errors: test error"));
         verify(controller, times(1)).updateErrorState(any());
@@ -565,7 +659,10 @@ public TransformState transform(Object source, TransformState prevState) {
         Metadata metadata = Metadata.builder().put(operatorMetadata).build();
         ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build();
 
-        var chunk = new ReservedStateChunk(Map.of("non-state", "two", "maker", "three"), new ReservedStateVersion(2L, Version.CURRENT));
+        var chunk = new ReservedStateChunk(
+            Map.of("non-state", "two", "maker", "three"),
+            new ReservedStateVersion(2L, BuildVersion.current())
+        );
         var orderedHandlers = List.of(exceptionThrower.name(), newStateMaker.name());
 
         ClusterService clusterService = mock(ClusterService.class);
diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java
index 9a2ab779669bc..1f453abf32303 100644
--- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java
+++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java
@@ -23,7 +23,15 @@
 
 public class ReservedStateUpdateTaskTests extends ESTestCase {
     public void testBlockedClusterState() {
-        var task = new ReservedStateUpdateTask("dummy", null, Map.of(), List.of(), e -> {}, ActionListener.noop());
+        var task = new ReservedStateUpdateTask(
+            "dummy",
+            null,
+            ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
+            Map.of(),
+            List.of(),
+            e -> {},
+            ActionListener.noop()
+        );
         ClusterState notRecoveredClusterState = ClusterState.builder(ClusterName.DEFAULT)
             .blocks(ClusterBlocks.builder().addGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK))
             .build();
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java
index b1b6a11764120..78cd90e8f5269 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
@@ -48,9 +49,9 @@ public void testMatchedTermsCount() throws IOException {
 
         // Partial match
         assertAllDocs(
-            Set.of(new Term("field", "foo"), new Term("field", "baz")),
+            Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")),
             ScriptTermStats::matchedTermsCount,
-            Map.of("doc-1", equalTo(1), "doc-2", equalTo(1), "doc-3", equalTo(0))
+            Map.of("doc-1", equalTo(2), "doc-2", equalTo(1), "doc-3", equalTo(0))
         );
 
         // Always returns 0 when no term is provided.
@@ -211,12 +212,12 @@ public void testTermFreq() throws IOException {
         // With missing terms
         {
             assertAllDocs(
-                Set.of(new Term("field", "foo"), new Term("field", "baz")),
+                Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")),
                 ScriptTermStats::termFreq,
                 Map.ofEntries(
-                    Map.entry("doc-1", equalTo(new StatsSummary(2, 1, 0, 1))),
-                    Map.entry("doc-2", equalTo(new StatsSummary(2, 2, 0, 2))),
-                    Map.entry("doc-3", equalTo(new StatsSummary(2, 0, 0, 0)))
+                    Map.entry("doc-1", equalTo(new StatsSummary(3, 2, 0, 1))),
+                    Map.entry("doc-2", equalTo(new StatsSummary(3, 2, 0, 2))),
+                    Map.entry("doc-3", equalTo(new StatsSummary(3, 0, 0, 0)))
                 )
             );
         }
@@ -274,10 +275,10 @@ public void testTermPositions() throws IOException {
         // With missing terms
         {
             assertAllDocs(
-                Set.of(new Term("field", "foo"), new Term("field", "baz")),
+                Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")),
                 ScriptTermStats::termPositions,
                 Map.ofEntries(
-                    Map.entry("doc-1", equalTo(new StatsSummary(1, 1, 1, 1))),
+                    Map.entry("doc-1", equalTo(new StatsSummary(2, 4, 1, 3))),
                     Map.entry("doc-2", equalTo(new StatsSummary(2, 3, 1, 2))),
                     Map.entry("doc-3", equalTo(new StatsSummary()))
                 )
@@ -311,7 +312,7 @@ private void withIndexSearcher(CheckedConsumer<IndexSearcher, IOException> consu
 
             Document doc = new Document();
             doc.add(new TextField("id", "doc-1", Field.Store.YES));
-            doc.add(new TextField("field", "foo bar", Field.Store.YES));
+            doc.add(new TextField("field", "foo bar qux", Field.Store.YES));
             w.addDocument(doc);
 
             doc = new Document();
@@ -336,10 +337,11 @@ private <T> void assertAllDocs(Set<Term> terms, Function<ScriptTermStats, T> fun
         withIndexSearcher(searcher -> {
             for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) {
                 IndexReader reader = leafReaderContext.reader();
+                StoredFields storedFields = reader.storedFields();
                 DocIdSetIterator docIdSetIterator = DocIdSetIterator.all(reader.maxDoc());
                 ScriptTermStats termStats = new ScriptTermStats(searcher, leafReaderContext, docIdSetIterator::docID, terms);
                 while (docIdSetIterator.nextDoc() <= reader.maxDoc()) {
-                    String docId = reader.document(docIdSetIterator.docID()).get("id");
+                    String docId = storedFields.document(docIdSetIterator.docID()).get("id");
                     if (expectedValues.containsKey(docId)) {
                         assertThat(function.apply(termStats), expectedValues.get(docId));
                     }
diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
index b716f11b5fffb..88c83df3e20fc 100644
--- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
@@ -89,7 +89,6 @@ protected SearchSourceBuilder createSearchSourceBuilder() {
         return RandomSearchRequestGenerator.randomSearchSourceBuilder(
             HighlightBuilderTests::randomHighlighterBuilder,
             SuggestBuilderTests::randomSuggestBuilder,
-            TestRankBuilder::randomRankBuilder,
             QueryRescorerBuilderTests::randomRescoreBuilder,
             randomExtBuilders,
             CollapseBuilderTests::randomCollapseBuilder,
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 0e4945e8bb8d1..a474c1dc38c50 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -81,6 +81,7 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.function.ToLongFunction;
@@ -507,10 +508,10 @@ public void testNewIdLoaderWithTsdbAndRoutingPathMatch() throws Exception {
         }
     }
 
-    public void testDetermineMaximumNumberOfSlices() {
+    private static ShardSearchRequest createParallelRequest() {
         IndexShard indexShard = mock(IndexShard.class);
         when(indexShard.shardId()).thenReturn(new ShardId("index", "uuid", 0));
-        ShardSearchRequest parallelReq = new ShardSearchRequest(
+        return new ShardSearchRequest(
             OriginalIndices.NONE,
             new SearchRequest().allowPartialSearchResults(randomBoolean()),
             indexShard.shardId(),
@@ -521,69 +522,74 @@ public void testDetermineMaximumNumberOfSlices() {
             System.currentTimeMillis(),
             null
         );
-        ShardSearchRequest singleSliceReq = new ShardSearchRequest(
-            OriginalIndices.NONE,
-            new SearchRequest().allowPartialSearchResults(randomBoolean())
-                .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME))),
-            indexShard.shardId(),
-            0,
-            1,
-            AliasFilter.EMPTY,
-            1f,
-            System.currentTimeMillis(),
-            null
-        );
-        int executorPoolSize = randomIntBetween(1, 100);
-        ExecutorService threadPoolExecutor = EsExecutors.newFixed(
-            "test",
-            executorPoolSize,
-            0,
-            Thread::new,
-            new ThreadContext(Settings.EMPTY),
-            EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
-        );
-        ExecutorService notThreadPoolExecutor = Executors.newWorkStealingPool();
-        ToLongFunction<String> fieldCardinality = name -> -1;
 
+    }
+
+    public void testDetermineMaximumNumberOfSlicesNoExecutor() {
+        ToLongFunction<String> fieldCardinality = name -> { throw new UnsupportedOperationException(); };
         assertEquals(
-            executorPoolSize,
+            1,
             DefaultSearchContext.determineMaximumNumberOfSlices(
-                threadPoolExecutor,
-                parallelReq,
+                null,
+                createParallelRequest(),
                 SearchService.ResultsType.DFS,
-                true,
+                randomBoolean(),
                 fieldCardinality
             )
         );
         assertEquals(
-            executorPoolSize,
+            1,
             DefaultSearchContext.determineMaximumNumberOfSlices(
-                threadPoolExecutor,
-                singleSliceReq,
-                SearchService.ResultsType.DFS,
-                true,
+                null,
+                createParallelRequest(),
+                SearchService.ResultsType.QUERY,
+                randomBoolean(),
                 fieldCardinality
             )
         );
+    }
+
+    public void testDetermineMaximumNumberOfSlicesNotThreadPoolExecutor() {
+        ExecutorService notThreadPoolExecutor = Executors.newWorkStealingPool();
+        ToLongFunction<String> fieldCardinality = name -> { throw new UnsupportedOperationException(); };
         assertEquals(
             1,
-            DefaultSearchContext.determineMaximumNumberOfSlices(null, parallelReq, SearchService.ResultsType.DFS, true, fieldCardinality)
+            DefaultSearchContext.determineMaximumNumberOfSlices(
+                notThreadPoolExecutor,
+                createParallelRequest(),
+                SearchService.ResultsType.DFS,
+                randomBoolean(),
+                fieldCardinality
+            )
         );
         assertEquals(
-            executorPoolSize,
+            1,
             DefaultSearchContext.determineMaximumNumberOfSlices(
-                threadPoolExecutor,
-                parallelReq,
+                notThreadPoolExecutor,
+                createParallelRequest(),
                 SearchService.ResultsType.QUERY,
-                true,
+                randomBoolean(),
                 fieldCardinality
             )
         );
+    }
+
+    public void testDetermineMaximumNumberOfSlicesEnableQueryPhaseParallelCollection() {
+        int executorPoolSize = randomIntBetween(1, 100);
+        ThreadPoolExecutor threadPoolExecutor = EsExecutors.newFixed(
+            "test",
+            executorPoolSize,
+            0,
+            Thread::new,
+            new ThreadContext(Settings.EMPTY),
+            EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+        );
+        ToLongFunction<String> fieldCardinality = name -> -1;
         assertEquals(
-            1,
+            executorPoolSize,
             DefaultSearchContext.determineMaximumNumberOfSlices(
                 threadPoolExecutor,
-                singleSliceReq,
+                createParallelRequest(),
                 SearchService.ResultsType.QUERY,
                 true,
                 fieldCardinality
@@ -592,54 +598,133 @@ public void testDetermineMaximumNumberOfSlices() {
         assertEquals(
             1,
             DefaultSearchContext.determineMaximumNumberOfSlices(
-                notThreadPoolExecutor,
-                parallelReq,
-                SearchService.ResultsType.DFS,
-                true,
+                threadPoolExecutor,
+                createParallelRequest(),
+                SearchService.ResultsType.QUERY,
+                false,
                 fieldCardinality
             )
         );
-
         assertEquals(
             executorPoolSize,
             DefaultSearchContext.determineMaximumNumberOfSlices(
                 threadPoolExecutor,
-                parallelReq,
+                createParallelRequest(),
                 SearchService.ResultsType.DFS,
-                false,
+                randomBoolean(),
                 fieldCardinality
             )
         );
-        assertEquals(
+    }
+
+    public void testDetermineMaximumNumberOfSlicesSingleSortByField() {
+        IndexShard indexShard = mock(IndexShard.class);
+        when(indexShard.shardId()).thenReturn(new ShardId("index", "uuid", 0));
+        ShardSearchRequest singleSliceReq = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            new SearchRequest().allowPartialSearchResults(randomBoolean())
+                .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME))),
+            indexShard.shardId(),
+            0,
             1,
-            DefaultSearchContext.determineMaximumNumberOfSlices(null, parallelReq, SearchService.ResultsType.DFS, false, fieldCardinality)
+            AliasFilter.EMPTY,
+            1f,
+            System.currentTimeMillis(),
+            null
         );
+        ToLongFunction<String> fieldCardinality = name -> { throw new UnsupportedOperationException(); };
+        int executorPoolSize = randomIntBetween(1, 100);
+        ThreadPoolExecutor threadPoolExecutor = EsExecutors.newFixed(
+            "test",
+            executorPoolSize,
+            0,
+            Thread::new,
+            new ThreadContext(Settings.EMPTY),
+            EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+        );
+        // DFS concurrency does not rely on slices, hence it kicks in regardless of the request (supportsParallelCollection is not called)
         assertEquals(
-            1,
+            executorPoolSize,
             DefaultSearchContext.determineMaximumNumberOfSlices(
                 threadPoolExecutor,
-                parallelReq,
-                SearchService.ResultsType.QUERY,
-                false,
+                singleSliceReq,
+                SearchService.ResultsType.DFS,
+                true,
                 fieldCardinality
             )
         );
-        assertEquals(
-            1,
-            DefaultSearchContext.determineMaximumNumberOfSlices(null, parallelReq, SearchService.ResultsType.QUERY, false, fieldCardinality)
-        );
         assertEquals(
             1,
             DefaultSearchContext.determineMaximumNumberOfSlices(
-                notThreadPoolExecutor,
-                parallelReq,
-                SearchService.ResultsType.DFS,
-                false,
+                threadPoolExecutor,
+                singleSliceReq,
+                SearchService.ResultsType.QUERY,
+                true,
                 fieldCardinality
             )
         );
     }
 
+    public void testDetermineMaximumNumberOfSlicesWithQueue() {
+        int executorPoolSize = randomIntBetween(1, 100);
+        ThreadPoolExecutor threadPoolExecutor = EsExecutors.newFixed(
+            "test",
+            executorPoolSize,
+            1000,
+            Thread::new,
+            new ThreadContext(Settings.EMPTY),
+            EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+        );
+        ToLongFunction<String> fieldCardinality = name -> { throw new UnsupportedOperationException(); };
+
+        for (int i = 0; i < executorPoolSize; i++) {
+            assertTrue(threadPoolExecutor.getQueue().offer(() -> {}));
+            assertEquals(
+                executorPoolSize,
+                DefaultSearchContext.determineMaximumNumberOfSlices(
+                    threadPoolExecutor,
+                    createParallelRequest(),
+                    SearchService.ResultsType.DFS,
+                    true,
+                    fieldCardinality
+                )
+            );
+            assertEquals(
+                executorPoolSize,
+                DefaultSearchContext.determineMaximumNumberOfSlices(
+                    threadPoolExecutor,
+                    createParallelRequest(),
+                    SearchService.ResultsType.QUERY,
+                    true,
+                    fieldCardinality
+                )
+            );
+        }
+        for (int i = 0; i < 100; i++) {
+            assertTrue(threadPoolExecutor.getQueue().offer(() -> {}));
+            assertEquals(
+                1,
+                DefaultSearchContext.determineMaximumNumberOfSlices(
+                    threadPoolExecutor,
+                    createParallelRequest(),
+                    SearchService.ResultsType.DFS,
+                    true,
+                    fieldCardinality
+                )
+            );
+            assertEquals(
+                1,
+                DefaultSearchContext.determineMaximumNumberOfSlices(
+                    threadPoolExecutor,
+                    createParallelRequest(),
+                    SearchService.ResultsType.QUERY,
+                    true,
+                    fieldCardinality
+                )
+            );
+        }
+    }
+
     public void testIsParallelCollectionSupportedForResults() {
         SearchSourceBuilder searchSourceBuilderOrNull = randomBoolean() ? null : new SearchSourceBuilder();
         ToLongFunction<String> fieldCardinality = name -> -1;
diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
index 6b42dbbb39c9f..5371893993318 100644
--- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
@@ -11,6 +11,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -173,8 +174,8 @@ public void testGeoTileFormat() {
         assertEquals("29/536869420/0", DocValueFormat.GEOTILE.format(longEncode(179.999, 89.999, 29)));
         assertEquals("29/1491/536870911", DocValueFormat.GEOTILE.format(longEncode(-179.999, -89.999, 29)));
         assertEquals("2/2/1", DocValueFormat.GEOTILE.format(longEncode(1, 1, 2)));
-        assertEquals("1/1/0", DocValueFormat.GEOTILE.format(longEncode(13, 95, 1)));
-        assertEquals("1/1/1", DocValueFormat.GEOTILE.format(longEncode(13, -95, 1)));
+        assertEquals("1/1/0", DocValueFormat.GEOTILE.format(longEncode(13, GeoUtils.normalizeLat(95), 1)));
+        assertEquals("1/1/1", DocValueFormat.GEOTILE.format(longEncode(13, GeoUtils.normalizeLat(-95), 1)));
     }
 
     public void testRawParse() {
diff --git a/server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java
index 6a6f5dc44ef6f..663b39d116913 100644
--- a/server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java
@@ -714,11 +714,8 @@ public void testMultiValuedOrds() throws Exception {
 
             @Override
             public long nextOrd() {
-                if (i < array[doc].length) {
-                    return array[doc][i++];
-                } else {
-                    return NO_MORE_ORDS;
-                }
+                assert i < array[doc].length;
+                return array[doc][i++];
             }
 
             @Override
@@ -762,7 +759,8 @@ private void verifySortedSet(Supplier<SortedSetDocValues> supplier, int maxDoc)
                 }
                 int expected = -1;
                 if (values.advanceExact(i)) {
-                    for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+                    for (int j = 0; j < values.docValueCount(); j++) {
+                        long ord = values.nextOrd();
                         if (expected == -1) {
                             expected = (int) ord;
                         } else {
@@ -810,7 +808,8 @@ private void verifySortedSet(
                         if (++count > maxChildren) {
                             break;
                         }
-                        for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+                        for (int i = 0; i < values.docValueCount(); i++) {
+                            long ord = values.nextOrd();
                             if (expected == -1) {
                                 expected = (int) ord;
                             } else {
diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
index 5e1296c354015..aa2e76f512cc8 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.PointValues;
@@ -105,14 +106,17 @@ public void testCancellableCollector() throws IOException {
             true
         );
 
-        Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+        Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()));
         assertThat(totalHits, equalTo(reader.numDocs()));
 
         searcher.addQueryCancellation(cancellation);
-        expectThrows(TaskCancelledException.class, () -> searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()));
+        expectThrows(
+            TaskCancelledException.class,
+            () -> searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()))
+        );
 
         searcher.removeQueryCancellation(cancellation);
-        Integer totalHits2 = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+        Integer totalHits2 = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()));
         assertThat(totalHits2, equalTo(reader.numDocs()));
     }
 
@@ -203,15 +207,17 @@ public void testExitableDirectoryReaderVectors() throws IOException {
         cancelled.set(false); // Avoid exception during construction of the wrapper objects
         FloatVectorValues vectorValues = searcher.getIndexReader().leaves().get(0).reader().getFloatVectorValues(KNN_FIELD_NAME);
         cancelled.set(true);
+        KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
         // On the first doc when already canceled, it throws
-        expectThrows(TaskCancelledException.class, vectorValues::nextDoc);
+        expectThrows(TaskCancelledException.class, iterator::nextDoc);
 
         cancelled.set(false); // Avoid exception during construction of the wrapper objects
         FloatVectorValues uncancelledVectorValues = searcher.getIndexReader().leaves().get(0).reader().getFloatVectorValues(KNN_FIELD_NAME);
+        uncancelledVectorValues.iterator();
         cancelled.set(true);
         searcher.removeQueryCancellation(cancellation);
         // On the first doc when already canceled, it throws, but with the cancellation removed, it should not
-        uncancelledVectorValues.nextDoc();
+        iterator.nextDoc();
     }
 
     private static class PointValuesIntersectVisitor implements PointValues.IntersectVisitor {
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 642804730a144..5dc07a41b3f8c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -772,7 +772,7 @@ public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId)
                 ),
             (response) -> {
                 SearchHits hits = response.getHits();
-                assertEquals(hits.getTotalHits().value, numDocs);
+                assertEquals(hits.getTotalHits().value(), numDocs);
                 assertEquals(hits.getHits().length, 2);
                 int index = 0;
                 for (SearchHit hit : hits.getHits()) {
@@ -2505,7 +2505,7 @@ public void testWaitOnRefresh() throws ExecutionException, InterruptedException
         );
         PlainActionFuture<Void> future = new PlainActionFuture<>();
         service.executeQueryPhase(request, task, future.delegateFailure((l, r) -> {
-            assertEquals(1, r.queryResult().getTotalHits().value);
+            assertEquals(1, r.queryResult().getTotalHits().value());
             l.onResponse(null);
         }));
         future.get();
@@ -2714,7 +2714,7 @@ public void testEnableSearchWorkerThreads() throws IOException {
             SearchShardTask task = new SearchShardTask(0, "type", "action", "description", null, emptyMap());
 
             try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.DFS, randomBoolean())) {
-                assertNotNull(searchContext.searcher().getExecutor());
+                assertTrue(searchContext.searcher().hasExecutor());
             }
 
             try {
@@ -2725,7 +2725,7 @@ public void testEnableSearchWorkerThreads() throws IOException {
                     .get();
                 assertTrue(response.isAcknowledged());
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.DFS, randomBoolean())) {
-                    assertNull(searchContext.searcher().getExecutor());
+                    assertFalse(searchContext.searcher().hasExecutor());
                 }
             } finally {
                 // reset original default setting
@@ -2735,7 +2735,7 @@ public void testEnableSearchWorkerThreads() throws IOException {
                     .setPersistentSettings(Settings.builder().putNull(SEARCH_WORKER_THREADS_ENABLED.getKey()).build())
                     .get();
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.DFS, randomBoolean())) {
-                    assertNotNull(searchContext.searcher().getExecutor());
+                    assertTrue(searchContext.searcher().hasExecutor());
                 }
             }
         }
@@ -2778,7 +2778,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
             {
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.DFS, true)) {
                     ContextIndexSearcher searcher = searchContext.searcher();
-                    assertNotNull(searcher.getExecutor());
+                    assertTrue(searcher.hasExecutor());
 
                     final int maxPoolSize = executor.getMaximumPoolSize();
                     assertEquals(
@@ -2795,7 +2795,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                     assertNotEquals("Sanity check to ensure this isn't the default of 1 when pool size is unset", 1, expectedSlices);
 
                     final long priorExecutorTaskCount = executor.getCompletedTaskCount();
-                    searcher.search(termQuery, new TotalHitCountCollectorManager());
+                    searcher.search(termQuery, new TotalHitCountCollectorManager(searcher.getSlices()));
                     assertBusy(
                         () -> assertEquals(
                             "DFS supports parallel collection, so the number of slices should be > 1.",
@@ -2808,7 +2808,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
             {
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.QUERY, true)) {
                     ContextIndexSearcher searcher = searchContext.searcher();
-                    assertNotNull(searcher.getExecutor());
+                    assertTrue(searcher.hasExecutor());
 
                     final int maxPoolSize = executor.getMaximumPoolSize();
                     assertEquals(
@@ -2825,7 +2825,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                     assertNotEquals("Sanity check to ensure this isn't the default of 1 when pool size is unset", 1, expectedSlices);
 
                     final long priorExecutorTaskCount = executor.getCompletedTaskCount();
-                    searcher.search(termQuery, new TotalHitCountCollectorManager());
+                    searcher.search(termQuery, new TotalHitCountCollectorManager(searcher.getSlices()));
                     assertBusy(
                         () -> assertEquals(
                             "QUERY supports parallel collection when enabled, so the number of slices should be > 1.",
@@ -2838,9 +2838,9 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
             {
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.FETCH, true)) {
                     ContextIndexSearcher searcher = searchContext.searcher();
-                    assertNull(searcher.getExecutor());
+                    assertFalse(searcher.hasExecutor());
                     final long priorExecutorTaskCount = executor.getCompletedTaskCount();
-                    searcher.search(termQuery, new TotalHitCountCollectorManager());
+                    searcher.search(termQuery, new TotalHitCountCollectorManager(searcher.getSlices()));
                     assertBusy(
                         () -> assertEquals(
                             "The number of slices should be 1 as FETCH does not support parallel collection and thus runs on the calling"
@@ -2854,9 +2854,9 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
             {
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.NONE, true)) {
                     ContextIndexSearcher searcher = searchContext.searcher();
-                    assertNull(searcher.getExecutor());
+                    assertFalse(searcher.hasExecutor());
                     final long priorExecutorTaskCount = executor.getCompletedTaskCount();
-                    searcher.search(termQuery, new TotalHitCountCollectorManager());
+                    searcher.search(termQuery, new TotalHitCountCollectorManager(searcher.getSlices()));
                     assertBusy(
                         () -> assertEquals(
                             "The number of slices should be 1 as NONE does not support parallel collection.",
@@ -2877,9 +2877,9 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                 {
                     try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.QUERY, true)) {
                         ContextIndexSearcher searcher = searchContext.searcher();
-                        assertNull(searcher.getExecutor());
+                        assertFalse(searcher.hasExecutor());
                         final long priorExecutorTaskCount = executor.getCompletedTaskCount();
-                        searcher.search(termQuery, new TotalHitCountCollectorManager());
+                        searcher.search(termQuery, new TotalHitCountCollectorManager(searcher.getSlices()));
                         assertBusy(
                             () -> assertEquals(
                                 "The number of slices should be 1 when QUERY parallel collection is disabled.",
@@ -2899,7 +2899,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                 {
                     try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.QUERY, true)) {
                         ContextIndexSearcher searcher = searchContext.searcher();
-                        assertNotNull(searcher.getExecutor());
+                        assertTrue(searcher.hasExecutor());
 
                         final int maxPoolSize = executor.getMaximumPoolSize();
                         assertEquals(
@@ -2916,7 +2916,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                         assertNotEquals("Sanity check to ensure this isn't the default of 1 when pool size is unset", 1, expectedSlices);
 
                         final long priorExecutorTaskCount = executor.getCompletedTaskCount();
-                        searcher.search(termQuery, new TotalHitCountCollectorManager());
+                        searcher.search(termQuery, new TotalHitCountCollectorManager(searcher.getSlices()));
                         assertBusy(
                             () -> assertEquals(
                                 "QUERY supports parallel collection when enabled, so the number of slices should be > 1.",
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/MultiBucketCollectorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/MultiBucketCollectorTests.java
index 6b4618bf3257a..ac5d886c9ba10 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/MultiBucketCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/MultiBucketCollectorTests.java
@@ -33,14 +33,8 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class MultiBucketCollectorTests extends ESTestCase {
-    private static class ScoreAndDoc extends Scorable {
+    private static class Score extends Scorable {
         float score;
-        int doc = -1;
-
-        @Override
-        public int docID() {
-            return doc;
-        }
 
         @Override
         public float score() {
@@ -247,7 +241,7 @@ public void testSetScorerAfterCollectionTerminated() throws IOException {
         collector1 = new TerminateAfterBucketCollector(collector1, 1);
         collector2 = new TerminateAfterBucketCollector(collector2, 2);
 
-        Scorable scorer = new ScoreAndDoc();
+        Scorable scorer = new Score();
 
         List<BucketCollector> collectors = Arrays.asList(collector1, collector2);
         Collections.shuffle(collectors, random());
@@ -275,4 +269,78 @@ public void testSetScorerAfterCollectionTerminated() throws IOException {
         assertFalse(setScorerCalled1.get());
         assertFalse(setScorerCalled2.get());
     }
+
+    public void testCacheScores() throws IOException {
+        ScoringBucketCollector scoringBucketCollector1 = new ScoringBucketCollector();
+        ScoringBucketCollector scoringBucketCollector2 = new ScoringBucketCollector();
+
+        DummyScorable scorable = new DummyScorable();
+
+        // First test the tester
+        LeafBucketCollector leafBucketCollector1 = scoringBucketCollector1.getLeafCollector(null);
+        LeafBucketCollector leafBucketCollector2 = scoringBucketCollector2.getLeafCollector(null);
+        leafBucketCollector1.setScorer(scorable);
+        leafBucketCollector2.setScorer(scorable);
+        leafBucketCollector1.collect(0, 0);
+        leafBucketCollector2.collect(0, 0);
+        assertEquals(2, scorable.numScoreCalls);
+
+        // reset
+        scorable.numScoreCalls = 0;
+        LeafBucketCollector leafBucketCollector = MultiBucketCollector.wrap(
+            randomBoolean(),
+            Arrays.asList(scoringBucketCollector1, scoringBucketCollector2)
+        ).getLeafCollector(null);
+        leafBucketCollector.setScorer(scorable);
+        leafBucketCollector.collect(0, 0);
+        // Even though both leaf collectors called scorable.score(), it only got called once thanks to caching
+        assertEquals(1, scorable.numScoreCalls);
+    }
+
+    private static class ScoringBucketCollector extends BucketCollector {
+        @Override
+        public ScoreMode scoreMode() {
+            return ScoreMode.COMPLETE; // needs scores
+        }
+
+        @Override
+        public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx) throws IOException {
+            return new ScoringLeafBucketCollector();
+        }
+
+        @Override
+        public void preCollection() throws IOException {
+
+        }
+
+        @Override
+        public void postCollection() throws IOException {
+
+        }
+    }
+
+    private static class ScoringLeafBucketCollector extends LeafBucketCollector {
+
+        private Scorable scorable;
+
+        @Override
+        public void setScorer(Scorable scorer) throws IOException {
+            this.scorable = scorer;
+        }
+
+        @Override
+        public void collect(int doc, long owningBucketOrd) throws IOException {
+            scorable.score();
+        }
+    }
+
+    private static class DummyScorable extends Scorable {
+        int numScoreCalls = 0;
+
+        @Override
+        public float score() throws IOException {
+            numScoreCalls++;
+            return 42f;
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
index 879c7e6aeff7f..eb5fa734a8c91 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
@@ -80,12 +80,12 @@ protected void indexData() throws Exception {
         indexRandom(true, docs);
 
         assertNoFailuresAndResponse(prepareSearch("idx").setRouting(routing1).setQuery(matchAllQuery()), resp -> {
-            long totalOnOne = resp.getHits().getTotalHits().value;
+            long totalOnOne = resp.getHits().getTotalHits().value();
             assertThat(totalOnOne, is(15L));
         });
         assertNoFailuresAndResponse(prepareSearch("idx").setRouting(routing2).setQuery(matchAllQuery()), resp -> {
             assertNoFailures(resp);
-            long totalOnTwo = resp.getHits().getTotalHits().value;
+            long totalOnTwo = resp.getHits().getTotalHits().value();
             assertThat(totalOnTwo, is(12L));
         });
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
index bbd12726ac4e3..28a032e7281e6 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
@@ -2571,19 +2571,19 @@ public void testWithKeywordAndTopHits() throws Exception {
             TopHits topHits = result.getBuckets().get(0).getAggregations().get("top_hits");
             assertNotNull(topHits);
             assertEquals(topHits.getHits().getHits().length, 2);
-            assertEquals(topHits.getHits().getTotalHits().value, 2L);
+            assertEquals(topHits.getHits().getTotalHits().value(), 2L);
             assertEquals("{keyword=c}", result.getBuckets().get(1).getKeyAsString());
             assertEquals(2L, result.getBuckets().get(1).getDocCount());
             topHits = result.getBuckets().get(1).getAggregations().get("top_hits");
             assertNotNull(topHits);
             assertEquals(topHits.getHits().getHits().length, 2);
-            assertEquals(topHits.getHits().getTotalHits().value, 2L);
+            assertEquals(topHits.getHits().getTotalHits().value(), 2L);
             assertEquals("{keyword=d}", result.getBuckets().get(2).getKeyAsString());
             assertEquals(1L, result.getBuckets().get(2).getDocCount());
             topHits = result.getBuckets().get(2).getAggregations().get("top_hits");
             assertNotNull(topHits);
             assertEquals(topHits.getHits().getHits().length, 1);
-            assertEquals(topHits.getHits().getTotalHits().value, 1L);
+            assertEquals(topHits.getHits().getTotalHits().value(), 1L);
         });
 
         testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new FieldExistsQuery("keyword")), dataset, () -> {
@@ -2598,13 +2598,13 @@ public void testWithKeywordAndTopHits() throws Exception {
             TopHits topHits = result.getBuckets().get(0).getAggregations().get("top_hits");
             assertNotNull(topHits);
             assertEquals(topHits.getHits().getHits().length, 2);
-            assertEquals(topHits.getHits().getTotalHits().value, 2L);
+            assertEquals(topHits.getHits().getTotalHits().value(), 2L);
             assertEquals("{keyword=d}", result.getBuckets().get(1).getKeyAsString());
             assertEquals(1L, result.getBuckets().get(1).getDocCount());
             topHits = result.getBuckets().get(1).getAggregations().get("top_hits");
             assertNotNull(topHits);
             assertEquals(topHits.getHits().getHits().length, 1);
-            assertEquals(topHits.getHits().getTotalHits().value, 1L);
+            assertEquals(topHits.getHits().getTotalHits().value(), 1L);
         });
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
index 71b93888ba243..8a72f8af7035c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
@@ -15,14 +15,29 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafMetaData;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.KnnCollector;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
@@ -54,8 +69,6 @@
 import static org.elasticsearch.index.mapper.NumberFieldMapper.NumberType.LONG;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
     static class ClassAndName {
@@ -71,11 +84,8 @@ static class ClassAndName {
     private IndexReader indexReader;
 
     @Before
-    public void setUpMocks() {
-        indexReader = mock(IndexReader.class);
-        IndexReaderContext indexReaderContext = mock(IndexReaderContext.class);
-        when(indexReaderContext.leaves()).thenReturn(List.of());
-        when(indexReader.getContext()).thenReturn(indexReaderContext);
+    public void set() {
+        indexReader = new DummyReader();
     }
 
     public void testRandomLong() throws IOException {
@@ -425,4 +435,126 @@ private static void createListCombinations(
             }
         }
     }
+
+    static class DummyReader extends LeafReader {
+        @Override
+        public CacheHelper getCoreCacheHelper() {
+            return null;
+        }
+
+        @Override
+        public Terms terms(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public NumericDocValues getNumericDocValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public SortedDocValues getSortedDocValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public NumericDocValues getNormValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public void searchNearestVectors(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException {
+
+        }
+
+        @Override
+        public void searchNearestVectors(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException {
+
+        }
+
+        @Override
+        public FieldInfos getFieldInfos() {
+            return null;
+        }
+
+        @Override
+        public Bits getLiveDocs() {
+            return null;
+        }
+
+        @Override
+        public PointValues getPointValues(String field) throws IOException {
+            return null;
+        }
+
+        @Override
+        public void checkIntegrity() throws IOException {
+
+        }
+
+        @Override
+        public LeafMetaData getMetaData() {
+            return null;
+        }
+
+        @Override
+        public TermVectors termVectors() throws IOException {
+            return null;
+        }
+
+        @Override
+        public int numDocs() {
+            return 0;
+        }
+
+        @Override
+        public int maxDoc() {
+            return 0;
+        }
+
+        @Override
+        public StoredFields storedFields() throws IOException {
+            return null;
+        }
+
+        @Override
+        protected void doClose() throws IOException {
+
+        }
+
+        @Override
+        public CacheHelper getReaderCacheHelper() {
+            return null;
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java
index 4b9a72bacc97d..3d1ed0704acf9 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java
@@ -11,6 +11,7 @@
 
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -258,7 +259,7 @@ public void testNumericSorted() {
     }
 
     private static IndexReader mockIndexReader(int maxDoc, int numDocs) {
-        IndexReader reader = mock(IndexReader.class);
+        IndexReader reader = mock(LeafReader.class);
         when(reader.hasDeletions()).thenReturn(maxDoc - numDocs > 0);
         when(reader.maxDoc()).thenReturn(maxDoc);
         when(reader.numDocs()).thenReturn(numDocs);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java
index b056cc1fcc988..975c1af3dc3d9 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
+import static org.elasticsearch.common.geo.GeoUtils.normalizeLat;
+import static org.elasticsearch.common.geo.GeoUtils.normalizeLon;
 import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM;
 import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange;
 import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint;
@@ -53,20 +55,20 @@ public void testLongEncode() {
         assertEquals(0x77FFFF4580000000L, longEncode(179.999, 89.999, 29));
         assertEquals(0x740000BA7FFFFFFFL, longEncode(-179.999, -89.999, 29));
         assertEquals(0x0800000040000001L, longEncode(1, 1, 2));
-        assertEquals(0x0C00000060000000L, longEncode(-20, 100, 3));
+        assertEquals(0x0C00000060000000L, longEncode(-20, normalizeLat(100), 3));
         assertEquals(0x71127D27C8ACA67AL, longEncode(13, -15, 28));
         assertEquals(0x4C0077776003A9ACL, longEncode(-12, 15, 19));
-        assertEquals(0x140000024000000EL, longEncode(-328.231870, 16.064082, 5));
-        assertEquals(0x6436F96B60000000L, longEncode(-590.769588, 89.549167, 25));
-        assertEquals(0x6411BD6BA0A98359L, longEncode(999.787079, 51.830093, 25));
-        assertEquals(0x751BD6BBCA983596L, longEncode(999.787079, 51.830093, 29));
-        assertEquals(0x77CF880A20000000L, longEncode(-557.039740, -632.103969, 29));
+        assertEquals(0x140000024000000EL, longEncode(normalizeLon(-328.231870), 16.064082, 5));
+        assertEquals(0x6436F96B60000000L, longEncode(normalizeLon(-590.769588), 89.549167, 25));
+        assertEquals(0x6411BD6BA0A98359L, longEncode(normalizeLon(999.787079), 51.830093, 25));
+        assertEquals(0x751BD6BBCA983596L, longEncode(normalizeLon(999.787079), 51.830093, 29));
+        assertEquals(0x77CF880A20000000L, longEncode(normalizeLon(-557.039740), normalizeLat(-632.103969), 29));
         assertEquals(0x7624FA4FA0000000L, longEncode(13, 88, 29));
         assertEquals(0x7624FA4FBFFFFFFFL, longEncode(13, -88, 29));
         assertEquals(0x0400000020000000L, longEncode(13, 89, 1));
         assertEquals(0x0400000020000001L, longEncode(13, -89, 1));
-        assertEquals(0x0400000020000000L, longEncode(13, 95, 1));
-        assertEquals(0x0400000020000001L, longEncode(13, -95, 1));
+        assertEquals(0x0400000020000000L, longEncode(13, normalizeLat(95), 1));
+        assertEquals(0x0400000020000001L, longEncode(13, normalizeLat(-95), 1));
 
         expectThrows(IllegalArgumentException.class, () -> longEncode(0, 0, -1));
         expectThrows(IllegalArgumentException.class, () -> longEncode(-1, 0, MAX_ZOOM + 1));
@@ -78,20 +80,20 @@ public void testLongEncodeFromString() {
         assertEquals(0x77FFFF4580000000L, longEncode(stringEncode(longEncode(179.999, 89.999, 29))));
         assertEquals(0x740000BA7FFFFFFFL, longEncode(stringEncode(longEncode(-179.999, -89.999, 29))));
         assertEquals(0x0800000040000001L, longEncode(stringEncode(longEncode(1, 1, 2))));
-        assertEquals(0x0C00000060000000L, longEncode(stringEncode(longEncode(-20, 100, 3))));
+        assertEquals(0x0C00000060000000L, longEncode(stringEncode(longEncode(-20, normalizeLat(100), 3))));
         assertEquals(0x71127D27C8ACA67AL, longEncode(stringEncode(longEncode(13, -15, 28))));
         assertEquals(0x4C0077776003A9ACL, longEncode(stringEncode(longEncode(-12, 15, 19))));
-        assertEquals(0x140000024000000EL, longEncode(stringEncode(longEncode(-328.231870, 16.064082, 5))));
-        assertEquals(0x6436F96B60000000L, longEncode(stringEncode(longEncode(-590.769588, 89.549167, 25))));
-        assertEquals(0x6411BD6BA0A98359L, longEncode(stringEncode(longEncode(999.787079, 51.830093, 25))));
-        assertEquals(0x751BD6BBCA983596L, longEncode(stringEncode(longEncode(999.787079, 51.830093, 29))));
-        assertEquals(0x77CF880A20000000L, longEncode(stringEncode(longEncode(-557.039740, -632.103969, 29))));
+        assertEquals(0x140000024000000EL, longEncode(stringEncode(longEncode(normalizeLon(-328.231870), 16.064082, 5))));
+        assertEquals(0x6436F96B60000000L, longEncode(stringEncode(longEncode(normalizeLon(-590.769588), 89.549167, 25))));
+        assertEquals(0x6411BD6BA0A98359L, longEncode(stringEncode(longEncode(normalizeLon(999.787079), 51.830093, 25))));
+        assertEquals(0x751BD6BBCA983596L, longEncode(stringEncode(longEncode(normalizeLon(999.787079), 51.830093, 29))));
+        assertEquals(0x77CF880A20000000L, longEncode(stringEncode(longEncode(normalizeLon(-557.039740), normalizeLat(-632.103969), 29))));
         assertEquals(0x7624FA4FA0000000L, longEncode(stringEncode(longEncode(13, 88, 29))));
         assertEquals(0x7624FA4FBFFFFFFFL, longEncode(stringEncode(longEncode(13, -88, 29))));
         assertEquals(0x0400000020000000L, longEncode(stringEncode(longEncode(13, 89, 1))));
         assertEquals(0x0400000020000001L, longEncode(stringEncode(longEncode(13, -89, 1))));
-        assertEquals(0x0400000020000000L, longEncode(stringEncode(longEncode(13, 95, 1))));
-        assertEquals(0x0400000020000001L, longEncode(stringEncode(longEncode(13, -95, 1))));
+        assertEquals(0x0400000020000000L, longEncode(stringEncode(longEncode(13, normalizeLat(95), 1))));
+        assertEquals(0x0400000020000001L, longEncode(stringEncode(longEncode(13, normalizeLat(-95), 1))));
 
         expectThrows(IllegalArgumentException.class, () -> longEncode("12/asdf/1"));
         expectThrows(IllegalArgumentException.class, () -> longEncode("foo"));
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
index 26e643510859c..06f1db352e8f0 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
@@ -42,9 +42,7 @@ public boolean advanceExact(int docID) {
 
         @Override
         public long nextOrd() {
-            if (i == ords.length) {
-                return NO_MORE_ORDS;
-            }
+            assert i < ords.length;
             return ords[i++];
         }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java
index 4a08295bd7bcd..48aabb61371e9 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java
@@ -414,7 +414,7 @@ public void testWithNestedScoringAggregations() throws IOException {
                                 InternalTopHits topHits = bucket.getAggregations().get("top_hits");
                                 TotalHits hits = topHits.getHits().getTotalHits();
                                 assertNotNull(hits);
-                                assertThat(hits.value, equalTo(counter));
+                                assertThat(hits.value(), equalTo(counter));
                                 assertThat(topHits.getHits().getMaxScore(), equalTo(Float.NaN));
                                 counter += 1;
                             }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index 2c76ed96da488..b267cb2e656b6 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -1730,8 +1730,8 @@ private void assertNestedTopHitsScore(InternalMultiBucketAggregation<?, ?> terms
         int ptr = 9;
         for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
             InternalTopHits topHits = bucket.getAggregations().get("top_hits");
-            assertThat(topHits.getHits().getTotalHits().value, equalTo((long) ptr));
-            assertEquals(TotalHits.Relation.EQUAL_TO, topHits.getHits().getTotalHits().relation);
+            assertThat(topHits.getHits().getTotalHits().value(), equalTo((long) ptr));
+            assertEquals(TotalHits.Relation.EQUAL_TO, topHits.getHits().getTotalHits().relation());
             if (withScore) {
                 assertThat(topHits.getHits().getMaxScore(), equalTo(1f));
             } else {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java
index 87eb2bdc29fbd..07d535167b318 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java
@@ -220,8 +220,8 @@ protected void assertReduced(InternalTopHits reduced, List<InternalTopHits> inpu
         TotalHits.Relation relation = TotalHits.Relation.EQUAL_TO;
         for (int input = 0; input < inputs.size(); input++) {
             SearchHits internalHits = inputs.get(input).getHits();
-            totalHits += internalHits.getTotalHits().value;
-            if (internalHits.getTotalHits().relation == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
+            totalHits += internalHits.getTotalHits().value();
+            if (internalHits.getTotalHits().relation() == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
                 relation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
             }
             maxScore = max(maxScore, internalHits.getMaxScore());
@@ -379,14 +379,14 @@ protected InternalTopHits mutateInstance(InternalTopHits instance) {
             case 2 -> size += between(1, 100);
             case 3 -> topDocs = new TopDocsAndMaxScore(
                 new TopDocs(
-                    new TotalHits(topDocs.topDocs.totalHits.value + between(1, 100), topDocs.topDocs.totalHits.relation),
+                    new TotalHits(topDocs.topDocs.totalHits.value() + between(1, 100), topDocs.topDocs.totalHits.relation()),
                     topDocs.topDocs.scoreDocs
                 ),
                 topDocs.maxScore + randomFloat()
             );
             case 4 -> {
                 TotalHits totalHits = new TotalHits(
-                    searchHits.getTotalHits().value + between(1, 100),
+                    searchHits.getTotalHits().value() + between(1, 100),
                     randomFrom(TotalHits.Relation.values())
                 );
                 searchHits = SearchHits.unpooled(searchHits.getHits(), totalHits, searchHits.getMaxScore() + randomFloat());
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java
index 0c5217ded982b..6fb147e3ffc89 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java
@@ -55,7 +55,7 @@ public void testTopLevel() throws Exception {
             result = testCase(query, topHits("_name"));
         }
         SearchHits searchHits = ((TopHits) result).getHits();
-        assertEquals(3L, searchHits.getTotalHits().value);
+        assertEquals(3L, searchHits.getTotalHits().value());
         assertEquals("3", searchHits.getAt(0).getId());
         assertEquals("2", searchHits.getAt(1).getId());
         assertEquals("1", searchHits.getAt(2).getId());
@@ -65,7 +65,7 @@ public void testTopLevel() throws Exception {
     public void testNoResults() throws Exception {
         TopHits result = (TopHits) testCase(new MatchNoDocsQuery(), topHits("_name").sort("string", SortOrder.DESC));
         SearchHits searchHits = result.getHits();
-        assertEquals(0L, searchHits.getTotalHits().value);
+        assertEquals(0L, searchHits.getTotalHits().value());
         assertFalse(AggregationInspectionHelper.hasValue(((InternalTopHits) result)));
     }
 
@@ -89,27 +89,27 @@ public void testInsideTerms() throws Exception {
         // The "a" bucket
         TopHits hits = (TopHits) terms.getBucketByKey("a").getAggregations().get("top");
         SearchHits searchHits = (hits).getHits();
-        assertEquals(2L, searchHits.getTotalHits().value);
+        assertEquals(2L, searchHits.getTotalHits().value());
         assertEquals("2", searchHits.getAt(0).getId());
         assertEquals("1", searchHits.getAt(1).getId());
         assertTrue(AggregationInspectionHelper.hasValue(((InternalTopHits) terms.getBucketByKey("a").getAggregations().get("top"))));
 
         // The "b" bucket
         searchHits = ((TopHits) terms.getBucketByKey("b").getAggregations().get("top")).getHits();
-        assertEquals(2L, searchHits.getTotalHits().value);
+        assertEquals(2L, searchHits.getTotalHits().value());
         assertEquals("3", searchHits.getAt(0).getId());
         assertEquals("1", searchHits.getAt(1).getId());
         assertTrue(AggregationInspectionHelper.hasValue(((InternalTopHits) terms.getBucketByKey("b").getAggregations().get("top"))));
 
         // The "c" bucket
         searchHits = ((TopHits) terms.getBucketByKey("c").getAggregations().get("top")).getHits();
-        assertEquals(1L, searchHits.getTotalHits().value);
+        assertEquals(1L, searchHits.getTotalHits().value());
         assertEquals("2", searchHits.getAt(0).getId());
         assertTrue(AggregationInspectionHelper.hasValue(((InternalTopHits) terms.getBucketByKey("c").getAggregations().get("top"))));
 
         // The "d" bucket
         searchHits = ((TopHits) terms.getBucketByKey("d").getAggregations().get("top")).getHits();
-        assertEquals(1L, searchHits.getTotalHits().value);
+        assertEquals(1L, searchHits.getTotalHits().value());
         assertEquals("3", searchHits.getAt(0).getId());
         assertTrue(AggregationInspectionHelper.hasValue(((InternalTopHits) terms.getBucketByKey("d").getAggregations().get("top"))));
     }
@@ -179,7 +179,7 @@ public void testSetScorer() throws Exception {
             .build();
         AggregationBuilder agg = AggregationBuilders.topHits("top_hits");
         TopHits result = searchAndReduce(reader, new AggTestConfig(agg, STRING_FIELD_TYPE).withQuery(query));
-        assertEquals(3, result.getHits().getTotalHits().value);
+        assertEquals(3, result.getHits().getTotalHits().value());
         reader.close();
         directory.close();
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
index e47614145e924..7fa2732191cd1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
@@ -88,12 +88,9 @@ public boolean advanceExact(int docID) {
 
             @Override
             public long nextOrd() {
-                if (consumed) {
-                    return SortedSetDocValues.NO_MORE_ORDS;
-                } else {
-                    consumed = true;
-                    return 0;
-                }
+                assert consumed == false;
+                consumed = true;
+                return 0;
             }
 
             @Override
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
index bc6b72d9ddd3e..2a36887cc459a 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
@@ -124,11 +124,8 @@ public long getValueCount() {
 
             @Override
             public long nextOrd() {
-                if (i < ords[doc].length) {
-                    return ords[doc][i++];
-                } else {
-                    return NO_MORE_ORDS;
-                }
+                assert i < ords[doc].length;
+                return ords[doc][i++];
             }
 
             @Override
@@ -153,10 +150,8 @@ public int docValueCount() {
                     for (int ord : ords[i]) {
                         assertEquals(values[ord], withMissingReplaced.lookupOrd(withMissingReplaced.nextOrd()));
                     }
-                    assertEquals(SortedSetDocValues.NO_MORE_ORDS, withMissingReplaced.nextOrd());
                 } else {
                     assertEquals(missing, withMissingReplaced.lookupOrd(withMissingReplaced.nextOrd()));
-                    assertEquals(SortedSetDocValues.NO_MORE_ORDS, withMissingReplaced.nextOrd());
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java
index caf8df55ce528..71fd3a4761cbe 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java
@@ -201,7 +201,10 @@ public void testCollectFromMiddle() throws IOException {
         BucketCollector collector = getBucketCollector(2 * DOC_COUNTS);
 
         // skip the first doc of segment 1 and 2
-        indexSearcher.search(SortedSetDocValuesField.newSlowSetQuery("_tsid", new BytesRef("tsid0"), new BytesRef("tsid1")), collector);
+        indexSearcher.search(
+            SortedSetDocValuesField.newSlowSetQuery("_tsid", List.of(new BytesRef("tsid0"), new BytesRef("tsid1"))),
+            collector
+        );
         collector.postCollection();
 
         reader.close();
diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 240a677f4cbfd..380b5189b3efc 100644
--- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -464,18 +463,6 @@ public void testToXContentWithPointInTime() throws IOException {
     }
 
     public void testParseIndicesBoost() throws IOException {
-        {
-            String restContent = """
-                { "indices_boost": {"foo": 1.0, "bar": 2.0}}""";
-            try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, restContent, RestApiVersion.V_7)) {
-                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false);
-                assertEquals(2, searchSourceBuilder.indexBoosts().size());
-                assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0));
-                assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1));
-                assertCriticalWarnings("Object format in indices_boost is deprecated, please use array format instead");
-            }
-        }
-
         {
             String restContent = """
                 {
@@ -554,16 +541,6 @@ public void testNegativeSizeErrors() throws IOException {
             );
             assertThat(ex.getMessage(), containsString(Integer.toString(boundedRandomSize)));
         }
-
-        restContent = "{\"size\" : -1}";
-        try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, restContent, RestApiVersion.V_7)) {
-            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false);
-            assertEquals(-1, searchSourceBuilder.size());
-        }
-        assertCriticalWarnings(
-            "Using search size of -1 is deprecated and will be removed in future versions. Instead, don't use the `size` "
-                + "parameter if you don't want to set it explicitly."
-        );
     }
 
     public void testNegativeTerminateAfter() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
index 138ee899dd906..3699cdee3912b 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -323,7 +324,7 @@ public MappedFieldType getFieldType(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(
                     name,
                     createDefaultIndexAnalyzers(),
-                    idxSettings.getMode().isSyntheticSourceEnabled()
+                    SourceFieldMapper.isSynthetic(idxSettings)
                 );
                 return builder.build(MapperBuilderContext.root(false, false)).fieldType();
             }
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
index 34ee0eec101b6..9957d8c92b955 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
@@ -28,7 +28,6 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.CollectorManager;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -47,6 +46,7 @@
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHitCountCollectorManager;
@@ -76,7 +76,6 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.IdentityHashMap;
@@ -251,7 +250,7 @@ public void testConcurrentCollection() throws Exception {
                     Integer.MAX_VALUE,
                     1
                 );
-                Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+                Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()));
                 assertEquals(numDocs, totalHits.intValue());
                 int numExpectedTasks = ContextIndexSearcher.computeSlices(searcher.getIndexReader().leaves(), Integer.MAX_VALUE, 1).length;
                 // check that each slice except for one that executes on the calling thread goes to the executor, no matter the queue size
@@ -367,7 +366,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
         assertEquals(1, searcher.count(new CreateScorerOnceQuery(new MatchAllDocsQuery())));
 
         TopDocs topDocs = searcher.search(new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "bar"))), 3f), 1);
-        assertEquals(1, topDocs.totalHits.value);
+        assertEquals(1, topDocs.totalHits.value());
         assertEquals(1, topDocs.scoreDocs.length);
         assertEquals(3f, topDocs.scoreDocs[0].score, 0);
 
@@ -406,7 +405,7 @@ private static void assertSlices(LeafSlice[] slices, int numDocs, int numThreads
         int sumDocs = 0;
         assertThat(slices.length, lessThanOrEqualTo(numThreads));
         for (LeafSlice slice : slices) {
-            int sliceDocs = Arrays.stream(slice.leaves).mapToInt(l -> l.reader().maxDoc()).sum();
+            int sliceDocs = slice.getMaxDocs();
             assertThat(sliceDocs, greaterThanOrEqualTo((int) (0.1 * numDocs)));
             sumDocs += sliceDocs;
         }
@@ -497,9 +496,14 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
                         }
                         return new ConstantScoreWeight(this, boost) {
                             @Override
-                            public Scorer scorer(LeafReaderContext context) {
+                            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                                 contextIndexSearcher.throwTimeExceededException();
-                                return new ConstantScoreScorer(this, score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc()));
+                                Scorer scorer = new ConstantScoreScorer(
+                                    score(),
+                                    scoreMode,
+                                    DocIdSetIterator.all(context.reader().maxDoc())
+                                );
+                                return new DefaultScorerSupplier(scorer);
                             }
 
                             @Override
@@ -583,7 +587,10 @@ public Query rewrite(IndexSearcher indexSearcher) {
                         return null;
                     }
                 };
-                Integer hitCount = contextIndexSearcher.search(testQuery, new TotalHitCountCollectorManager());
+                Integer hitCount = contextIndexSearcher.search(
+                    testQuery,
+                    new TotalHitCountCollectorManager(contextIndexSearcher.getSlices())
+                );
                 assertEquals(0, hitCount.intValue());
                 assertTrue(contextIndexSearcher.timeExceeded());
             } finally {
@@ -747,15 +754,9 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
         }
 
         @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            assertTrue(seenLeaves.add(context.reader().getCoreCacheHelper().getKey()));
-            return weight.scorer(context);
-        }
-
-        @Override
-        public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
+        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
             assertTrue(seenLeaves.add(context.reader().getCoreCacheHelper().getKey()));
-            return weight.bulkScorer(context);
+            return weight.scorerSupplier(context);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java b/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java
index 370584e3f29f5..bddfd53b2b120 100644
--- a/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java
+++ b/server/src/test/java/org/elasticsearch/search/lookup/SourceFilterTests.java
@@ -112,4 +112,48 @@ public Source filter(SourceFilter sourceFilter) {
 
     }
 
+    // Verification for issue #109668
+    public void testIncludeParentAndExcludeChildEmptyArray() {
+        Source fromMap = Source.fromMap(Map.of("myArray", List.of()), XContentType.JSON);
+        Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" }));
+        assertEquals(filteredMap.source(), Map.of("myArray", List.of()));
+        Source fromBytes = Source.fromBytes(new BytesArray("{\"myArray\": []}"), XContentType.JSON);
+        Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" }));
+        assertEquals(filteredBytes.source(), Map.of("myArray", List.of()));
+    }
+
+    public void testIncludeParentAndExcludeChildEmptyObject() {
+        Source fromMap = Source.fromMap(Map.of("myObject", Map.of()), XContentType.JSON);
+        Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" }));
+        assertEquals(filteredMap.source(), Map.of("myObject", Map.of()));
+        Source fromBytes = Source.fromBytes(new BytesArray("{\"myObject\": {}}"), XContentType.JSON);
+        Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" }));
+        assertEquals(filteredBytes.source(), Map.of("myObject", Map.of()));
+    }
+
+    public void testIncludeParentAndExcludeChildSubFieldsArrays() {
+        Source fromMap = Source.fromMap(
+            Map.of("myArray", List.of(Map.<String, Object>of("myField", "myValue", "other", "otherValue"))),
+            XContentType.JSON
+        );
+        Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" }));
+        assertEquals(filteredMap.source(), Map.of("myArray", List.of(Map.of("other", "otherValue"))));
+        Source fromBytes = Source.fromBytes(new BytesArray("""
+            { "myArray": [ { "myField": "myValue", "other": "otherValue" } ] }"""), XContentType.JSON);
+        Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myArray" }, new String[] { "myArray.myField" }));
+        assertEquals(filteredBytes.source(), Map.of("myArray", List.of(Map.of("other", "otherValue"))));
+    }
+
+    public void testIncludeParentAndExcludeChildSubFieldsObjects() {
+        Source fromMap = Source.fromMap(
+            Map.of("myObject", Map.<String, Object>of("myField", "myValue", "other", "otherValue")),
+            XContentType.JSON
+        );
+        Source filteredMap = fromMap.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" }));
+        assertEquals(filteredMap.source(), Map.of("myObject", Map.of("other", "otherValue")));
+        Source fromBytes = Source.fromBytes(new BytesArray("""
+            { "myObject": { "myField": "myValue", "other": "otherValue" } }"""), XContentType.JSON);
+        Source filteredBytes = fromBytes.filter(new SourceFilter(new String[] { "myObject" }, new String[] { "myObject.myField" }));
+        assertEquals(filteredBytes.source(), Map.of("myObject", Map.of("other", "otherValue")));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java
index c42f3156c6d29..b728d40900570 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileCollectorManagerTests.java
@@ -125,14 +125,14 @@ public void testManagerWithSearcher() throws IOException {
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(10, null, 1000);
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
         }
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(10, null, 1000);
             String profileReason = "profiler_reason";
             ProfileCollectorManager<TopDocs> profileCollectorManager = new ProfileCollectorManager<>(topDocsManager, profileReason);
             TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), profileCollectorManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             CollectorResult result = profileCollectorManager.getCollectorTree();
             assertEquals("profiler_reason", result.getReason());
             assertEquals("SimpleTopScoreDocCollector", result.getName());
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java
index e868293ef4a1c..98d79df63db8e 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java
@@ -10,14 +10,12 @@
 package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Matches;
 import org.apache.lucene.search.MatchesIterator;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
@@ -34,10 +32,6 @@ private static class FakeScorer extends Scorer {
 
         public float maxScore, minCompetitiveScore;
 
-        protected FakeScorer(Weight weight) {
-            super(weight);
-        }
-
         @Override
         public DocIdSetIterator iterator() {
             throw new UnsupportedOperationException();
@@ -75,22 +69,14 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
             return Explanation.match(1, "fake_description");
         }
 
-        @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            FakeScorer fakeScorer = new FakeScorer(this);
-            fakeScorer.maxScore = 42f;
-            return fakeScorer;
-        }
-
         @Override
         public ScorerSupplier scorerSupplier(LeafReaderContext context) {
-            Weight weight = this;
             return new ScorerSupplier() {
                 private long cost = 0;
 
                 @Override
                 public Scorer get(long leadCost) {
-                    return new Scorer(weight) {
+                    return new Scorer() {
                         @Override
                         public DocIdSetIterator iterator() {
                             return null;
@@ -187,23 +173,17 @@ public Iterator<String> iterator() {
     }
 
     public void testPropagateMinCompetitiveScore() throws IOException {
-        Query query = new MatchAllDocsQuery();
-        Weight weight = query.createWeight(newSearcher(new MultiReader()), ScoreMode.TOP_SCORES, 1f);
-        FakeScorer fakeScorer = new FakeScorer(weight);
+        FakeScorer fakeScorer = new FakeScorer();
         QueryProfileBreakdown profile = new QueryProfileBreakdown();
-        ProfileWeight profileWeight = new ProfileWeight(query, weight, profile);
-        ProfileScorer profileScorer = new ProfileScorer(profileWeight, fakeScorer, profile);
+        ProfileScorer profileScorer = new ProfileScorer(fakeScorer, profile);
         profileScorer.setMinCompetitiveScore(0.42f);
         assertEquals(0.42f, fakeScorer.minCompetitiveScore, 0f);
     }
 
     public void testPropagateMaxScore() throws IOException {
-        Query query = new MatchAllDocsQuery();
-        Weight weight = query.createWeight(newSearcher(new MultiReader()), ScoreMode.TOP_SCORES, 1f);
-        FakeScorer fakeScorer = new FakeScorer(weight);
+        FakeScorer fakeScorer = new FakeScorer();
         QueryProfileBreakdown profile = new QueryProfileBreakdown();
-        ProfileWeight profileWeight = new ProfileWeight(query, weight, profile);
-        ProfileScorer profileScorer = new ProfileScorer(profileWeight, fakeScorer, profile);
+        ProfileScorer profileScorer = new ProfileScorer(fakeScorer, profile);
         profileScorer.setMinCompetitiveScore(0.42f);
         fakeScorer.maxScore = 42f;
         assertEquals(42f, profileScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
index 9df03905f7be2..44c46e3f692ba 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
@@ -241,11 +241,6 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                     throw new UnsupportedOperationException();
                 }
 
-                @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
-                    throw new UnsupportedOperationException();
-                }
-
                 @Override
                 public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                     return new ScorerSupplier() {
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseCollectorTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseCollectorTests.java
index d33c033ecef2d..de6218e912953 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseCollectorTests.java
@@ -120,7 +120,7 @@ public void testTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(new MatchAllDocsQuery(), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numDocs, result.topDocs.totalHits.value);
+            assertEquals(numDocs, result.topDocs.totalHits.value());
         }
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topScoreDocManager = new TopScoreDocCollectorManager(1, null, 1000);
@@ -133,7 +133,7 @@ public void testTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(new TermQuery(new Term("field2", "value")), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
         }
     }
 
@@ -150,7 +150,7 @@ public void testWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(new MatchAllDocsQuery(), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numDocs, result.topDocs.totalHits.value);
+            assertEquals(numDocs, result.topDocs.totalHits.value());
             assertEquals(numDocs, result.aggs.intValue());
         }
         {
@@ -165,7 +165,7 @@ public void testWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(new TermQuery(new Term("field2", "value")), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
             assertEquals(numField2Docs, result.aggs.intValue());
         }
     }
@@ -184,7 +184,7 @@ public void testPostFilterTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(new MatchAllDocsQuery(), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
         }
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(1, null, 1000);
@@ -199,7 +199,7 @@ public void testPostFilterTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(new MatchAllDocsQuery(), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numDocs, result.topDocs.totalHits.value);
+            assertEquals(numDocs, result.topDocs.totalHits.value());
         }
     }
 
@@ -218,7 +218,7 @@ public void testPostFilterWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(new MatchAllDocsQuery(), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numDocs, result.topDocs.totalHits.value);
+            assertEquals(numDocs, result.topDocs.totalHits.value());
             assertEquals(numDocs, result.aggs.intValue());
         }
         {
@@ -235,7 +235,7 @@ public void testPostFilterWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(new MatchAllDocsQuery(), manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
             // post_filter is not applied to aggs
             assertEquals(reader.maxDoc(), result.aggs.intValue());
         }
@@ -251,7 +251,7 @@ public void testMinScoreTopDocsOnly() throws IOException {
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField2Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
             thresholdScore = topDocs.scoreDocs[numField2Docs].score;
         }
@@ -266,7 +266,7 @@ public void testMinScoreTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
         }
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(1, null, 1000);
@@ -279,7 +279,7 @@ public void testMinScoreTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numDocs, result.topDocs.totalHits.value);
+            assertEquals(numDocs, result.topDocs.totalHits.value());
         }
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(1, null, 1000);
@@ -292,7 +292,7 @@ public void testMinScoreTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(0, result.topDocs.totalHits.value);
+            assertEquals(0, result.topDocs.totalHits.value());
         }
     }
 
@@ -306,7 +306,7 @@ public void testMinScoreWithAggs() throws IOException {
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField2Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
             thresholdScore = topDocs.scoreDocs[numField2Docs].score;
         }
@@ -322,7 +322,7 @@ public void testMinScoreWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
             // min_score is applied to aggs as well as top docs
             assertEquals(numField2Docs, result.aggs.intValue());
         }
@@ -338,7 +338,7 @@ public void testMinScoreWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numDocs, result.topDocs.totalHits.value);
+            assertEquals(numDocs, result.topDocs.totalHits.value());
             assertEquals(numDocs, result.aggs.intValue());
         }
         {
@@ -353,7 +353,7 @@ public void testMinScoreWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(0, result.topDocs.totalHits.value);
+            assertEquals(0, result.topDocs.totalHits.value());
             assertEquals(0, result.aggs.intValue());
         }
     }
@@ -370,7 +370,7 @@ public void testPostFilterAndMinScoreTopDocsOnly() throws IOException {
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField3Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
             thresholdScore = topDocs.scoreDocs[numField3Docs].score;
         }
@@ -385,7 +385,7 @@ public void testPostFilterAndMinScoreTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2AndField3Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2AndField3Docs, result.topDocs.totalHits.value());
         }
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(1, null, 1000);
@@ -398,7 +398,7 @@ public void testPostFilterAndMinScoreTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
         }
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(1, null, 1000);
@@ -411,7 +411,7 @@ public void testPostFilterAndMinScoreTopDocsOnly() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(0, result.topDocs.totalHits.value);
+            assertEquals(0, result.topDocs.totalHits.value());
         }
     }
 
@@ -427,7 +427,7 @@ public void testPostFilterAndMinScoreWithAggs() throws IOException {
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField3Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
             thresholdScore = topDocs.scoreDocs[numField3Docs].score;
         }
@@ -443,7 +443,7 @@ public void testPostFilterAndMinScoreWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2AndField3Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2AndField3Docs, result.topDocs.totalHits.value());
             assertEquals(numField3Docs, result.aggs.intValue());
         }
         {
@@ -458,7 +458,7 @@ public void testPostFilterAndMinScoreWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(numField2Docs, result.topDocs.totalHits.value);
+            assertEquals(numField2Docs, result.topDocs.totalHits.value());
             assertEquals(numDocs, result.aggs.intValue());
         }
         {
@@ -473,7 +473,7 @@ public void testPostFilterAndMinScoreWithAggs() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertFalse(result.terminatedAfter);
-            assertEquals(0, result.topDocs.totalHits.value);
+            assertEquals(0, result.topDocs.totalHits.value());
             assertEquals(0, result.aggs.intValue());
         }
     }
@@ -623,7 +623,7 @@ public void testTerminateAfterTopDocsOnlyWithMinScore() throws IOException {
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField2Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
         }
         {
@@ -638,7 +638,7 @@ public void testTerminateAfterTopDocsOnlyWithMinScore() throws IOException {
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertTrue(result.terminatedAfter);
-            assertEquals(terminateAfter, result.topDocs.totalHits.value);
+            assertEquals(terminateAfter, result.topDocs.totalHits.value());
         }
     }
 
@@ -651,7 +651,7 @@ public void testTerminateAfterWithAggsAndMinScore() throws IOException {
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField2Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
         }
         {
@@ -667,7 +667,7 @@ public void testTerminateAfterWithAggsAndMinScore() throws IOException {
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertTrue(result.terminatedAfter);
-            assertEquals(terminateAfter, result.topDocs.totalHits.value);
+            assertEquals(terminateAfter, result.topDocs.totalHits.value());
             assertEquals(terminateAfter, result.aggs.intValue());
         }
     }
@@ -683,7 +683,7 @@ public void testTerminateAfterAndPostFilterAndMinScoreTopDocsOnly() throws IOExc
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField3Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
         }
         {
@@ -698,7 +698,7 @@ public void testTerminateAfterAndPostFilterAndMinScoreTopDocsOnly() throws IOExc
             );
             Result<TopDocs, Void> result = searcher.search(booleanQuery, manager);
             assertTrue(result.terminatedAfter);
-            assertEquals(terminateAfter, result.topDocs.totalHits.value);
+            assertEquals(terminateAfter, result.topDocs.totalHits.value());
         }
     }
 
@@ -713,7 +713,7 @@ public void testTerminateAfterAndPostFilterAndMinScoreWithAggs() throws IOExcept
         {
             CollectorManager<TopScoreDocCollector, TopDocs> topDocsManager = new TopScoreDocCollectorManager(numField3Docs + 1, null, 1000);
             TopDocs topDocs = searcher.search(booleanQuery, topDocsManager);
-            assertEquals(numDocs, topDocs.totalHits.value);
+            assertEquals(numDocs, topDocs.totalHits.value());
             maxScore = topDocs.scoreDocs[0].score;
         }
         {
@@ -729,7 +729,7 @@ public void testTerminateAfterAndPostFilterAndMinScoreWithAggs() throws IOExcept
             );
             Result<TopDocs, Integer> result = searcher.search(booleanQuery, manager);
             assertTrue(result.terminatedAfter);
-            assertEquals(terminateAfter, result.topDocs.totalHits.value);
+            assertEquals(terminateAfter, result.topDocs.totalHits.value());
             // aggs see more documents because the filter is not applied to them
             assertThat(result.aggs, Matchers.greaterThanOrEqualTo(terminateAfter));
         }
@@ -1139,11 +1139,6 @@ public void testSetScorerAfterCollectionTerminated() throws IOException {
             public float score() {
                 return 0;
             }
-
-            @Override
-            public int docID() {
-                return 0;
-            }
         };
 
         QueryPhaseCollector queryPhaseCollector = new QueryPhaseCollector(
@@ -1473,11 +1468,6 @@ public float score() throws IOException {
             return 0;
         }
 
-        @Override
-        public int docID() {
-            return 0;
-        }
-
         @Override
         public void setMinCompetitiveScore(float minScore) {
             setMinCompetitiveScoreCalled = true;
@@ -1521,7 +1511,7 @@ public void setScorer(Scorable scorer) throws IOException {
             setScorerCalled = true;
             if (expectedScorable != null) {
                 while (expectedScorable.equals(scorer.getClass()) == false && scorer instanceof FilterScorable) {
-                    scorer = scorer.getChildren().iterator().next().child;
+                    scorer = scorer.getChildren().iterator().next().child();
                 }
                 assertEquals(expectedScorable, scorer.getClass());
             }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index 56a8b0f3a8c30..1f74668158e0e 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -144,7 +144,7 @@ private void countTestCase(Query query, IndexReader reader, boolean shouldCollec
             QueryPhase.addCollectorsAndSearch(context);
 
             ContextIndexSearcher countSearcher = shouldCollectCount ? newContextSearcher(reader) : noCollectionContextSearcher(reader);
-            assertEquals(countSearcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value);
+            assertEquals(countSearcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value());
         }
     }
 
@@ -233,15 +233,15 @@ public void testPostFilterDisablesHitCountShortcut() throws Exception {
         try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) {
             context.setSize(0);
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
         try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 10), new MatchAllDocsQuery())) {
             // shortcutTotalHitCount makes us not track total hits as part of the top docs collection, hence size is the threshold
             context.setSize(10);
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
         try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) {
             // QueryPhaseCollector does not propagate Weight#count when a post_filter is provided, hence it forces collection despite
@@ -249,16 +249,16 @@ public void testPostFilterDisablesHitCountShortcut() throws Exception {
             context.setSize(0);
             context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery()));
             QueryPhase.executeQuery(context);
-            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
         try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) {
             // shortcutTotalHitCount is disabled for filter collectors, hence we collect until track_total_hits
             context.setSize(10);
             context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery()));
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
     }
 
@@ -269,8 +269,8 @@ public void testTerminateAfterWithFilter() throws Exception {
             context.setSize(10);
             context.parsedPostFilter(new ParsedQuery(new TermQuery(new Term("foo", "bar"))));
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(1, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(1, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
         }
     }
@@ -280,15 +280,15 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception {
         try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) {
             context.setSize(0);
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
         try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 10), new MatchAllDocsQuery())) {
             // shortcutTotalHitCount makes us not track total hits as part of the top docs collection, hence size is the threshold
             context.setSize(10);
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
         try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) {
             // QueryPhaseCollector does not propagate Weight#count when min_score is provided, hence it forces collection despite
@@ -296,16 +296,16 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception {
             context.setSize(0);
             context.minimumScore(100);
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
         try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) {
             // shortcutTotalHitCount is disabled for filter collectors, hence we collect until track_total_hits
             context.setSize(10);
             context.minimumScore(100);
             QueryPhase.executeQuery(context);
-            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
         }
     }
 
@@ -336,17 +336,17 @@ public void testInOrderScrollOptimization() throws Exception {
             context.setSize(size);
 
             QueryPhase.addCollectorsAndSearch(context);
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
             assertNull(context.queryResult().terminatedEarly());
             assertThat(context.terminateAfter(), equalTo(0));
-            assertThat(context.queryResult().getTotalHits().value, equalTo((long) numDocs));
+            assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs));
 
             context.setSearcher(earlyTerminationContextSearcher(reader, size));
             QueryPhase.addCollectorsAndSearch(context);
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertThat(context.queryResult().getTotalHits().value, equalTo((long) numDocs));
-            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs));
+            assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation());
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0].doc, greaterThanOrEqualTo(size));
         }
     }
@@ -364,8 +364,8 @@ public void testTerminateAfterSize0HitCountShortcut() throws Exception {
             context.setSize(0);
             QueryPhase.addCollectorsAndSearch(context);
             assertFalse(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
         }
         // test interaction between trackTotalHits and terminateAfter
@@ -375,8 +375,8 @@ public void testTerminateAfterSize0HitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(-1);
             QueryPhase.executeQuery(context);
             assertFalse(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(0L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(0L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
         }
         try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) {
@@ -387,8 +387,8 @@ public void testTerminateAfterSize0HitCountShortcut() throws Exception {
             QueryPhase.executeQuery(context);
             assertFalse(context.queryResult().terminatedEarly());
             // Given that total hit count does not require collection, PartialHitCountCollector does not early terminate.
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
         }
     }
@@ -407,8 +407,8 @@ public void testTerminateAfterSize0NoHitCountShortcut() throws Exception {
             context.setSize(0);
             QueryPhase.executeQuery(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(1L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
         }
         // test interaction between trackTotalHits and terminateAfter
@@ -419,8 +419,8 @@ public void testTerminateAfterSize0NoHitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(-1);
             QueryPhase.executeQuery(context);
             assertFalse(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(0L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(0L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
         }
         {
@@ -434,9 +434,9 @@ public void testTerminateAfterSize0NoHitCountShortcut() throws Exception {
                 context.trackTotalHitsUpTo(trackTotalHits);
                 QueryPhase.executeQuery(context);
                 assertFalse(context.queryResult().terminatedEarly());
-                assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) trackTotalHits));
+                assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) trackTotalHits));
                 assertThat(
-                    context.queryResult().topDocs().topDocs.totalHits.relation,
+                    context.queryResult().topDocs().topDocs.totalHits.relation(),
                     equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)
                 );
                 assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
@@ -449,8 +449,8 @@ public void testTerminateAfterSize0NoHitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(randomIntBetween(11, Integer.MAX_VALUE));
             QueryPhase.executeQuery(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(10L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(10L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
         }
     }
@@ -468,7 +468,7 @@ public void testTerminateAfterWithHitsHitCountShortcut() throws Exception {
             context.setSize(10);
             QueryPhase.executeQuery(context);
             assertFalse(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(10));
         }
         try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 1), new MatchAllDocsQuery())) {
@@ -477,8 +477,8 @@ public void testTerminateAfterWithHitsHitCountShortcut() throws Exception {
             context.setSize(1);
             QueryPhase.addCollectorsAndSearch(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
         }
         // test interaction between trackTotalHits and terminateAfter
@@ -489,8 +489,8 @@ public void testTerminateAfterWithHitsHitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(-1);
             QueryPhase.executeQuery(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(0L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(0L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(7));
         }
         try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 7), new MatchAllDocsQuery())) {
@@ -500,8 +500,8 @@ public void testTerminateAfterWithHitsHitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(randomIntBetween(1, Integer.MAX_VALUE));
             QueryPhase.executeQuery(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(7));
         }
         {
@@ -515,8 +515,8 @@ public void testTerminateAfterWithHitsHitCountShortcut() throws Exception {
                 context.trackTotalHitsUpTo(randomIntBetween(1, Integer.MAX_VALUE));
                 QueryPhase.executeQuery(context);
                 assertThat(context.queryResult().terminatedEarly(), either(is(true)).or(is(false)));
-                assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-                assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+                assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+                assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
                 assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(size));
             }
         }
@@ -535,8 +535,8 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception {
             context.setSize(1);
             QueryPhase.addCollectorsAndSearch(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(1L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
         }
         // test interaction between trackTotalHits and terminateAfter
@@ -546,8 +546,8 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(-1);
             QueryPhase.executeQuery(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(0L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(0L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(7));
         }
         try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 7), query)) {
@@ -558,8 +558,8 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(randomIntBetween(1, 6));
             QueryPhase.executeQuery(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(7L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(7L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(7));
         }
         try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 7), query)) {
@@ -572,8 +572,8 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception {
             // depending on docs distribution we may or may not be able to honor terminate_after: low scoring hits are skipped via
             // setMinCompetitiveScore, which bypasses terminate_after until the next leaf collector is pulled, when that happens.
             assertThat(context.queryResult().terminatedEarly(), either(is(true)).or(is(false)));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(7L));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(7L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(5));
         }
         try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 7), query)) {
@@ -584,9 +584,9 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception {
             context.trackTotalHitsUpTo(randomIntBetween(8, Integer.MAX_VALUE));
             QueryPhase.executeQuery(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(7L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(7L));
             // TODO this looks off, it should probably be GREATER_THAN_OR_EQUAL_TO
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(7));
         }
     }
@@ -599,8 +599,8 @@ public void testIndexSortingEarlyTermination() throws Exception {
             context.setSize(1);
             context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW }));
             QueryPhase.addCollectorsAndSearch(context);
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class));
             FieldDoc fieldDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[0];
@@ -612,7 +612,7 @@ public void testIndexSortingEarlyTermination() throws Exception {
             context.parsedPostFilter(new ParsedQuery(new MinDocQuery(1)));
             QueryPhase.addCollectorsAndSearch(context);
             assertNull(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(numDocs - 1L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(numDocs - 1L));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class));
             FieldDoc fieldDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[0];
@@ -623,7 +623,7 @@ public void testIndexSortingEarlyTermination() throws Exception {
             context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW }));
             QueryPhase.executeQuery(context);
             assertNull(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class));
             FieldDoc fieldDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[0];
@@ -686,18 +686,18 @@ public void testIndexSortScrollOptimization() throws Exception {
                 context.sort(searchSortAndFormat);
 
                 QueryPhase.addCollectorsAndSearch(context);
-                assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
+                assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
                 assertNull(context.queryResult().terminatedEarly());
                 assertThat(context.terminateAfter(), equalTo(0));
-                assertThat(context.queryResult().getTotalHits().value, equalTo((long) numDocs));
+                assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs));
                 int sizeMinus1 = context.queryResult().topDocs().topDocs.scoreDocs.length - 1;
                 FieldDoc lastDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[sizeMinus1];
                 context.setSearcher(earlyTerminationContextSearcher(reader, 10));
                 QueryPhase.addCollectorsAndSearch(context);
                 assertNull(context.queryResult().terminatedEarly());
-                assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
+                assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
                 assertThat(context.terminateAfter(), equalTo(0));
-                assertThat(context.queryResult().getTotalHits().value, equalTo((long) numDocs));
+                assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs));
                 FieldDoc firstDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[0];
                 for (int i = 0; i < searchSortAndFormat.sort.getSort().length; i++) {
                     @SuppressWarnings("unchecked")
@@ -746,8 +746,8 @@ public void testDisableTopScoreCollection() throws Exception {
             );
             assertEquals(collectorManager.newCollector().scoreMode(), org.apache.lucene.search.ScoreMode.COMPLETE);
             QueryPhase.executeQuery(context);
-            assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value);
-            assertEquals(context.queryResult().topDocs().topDocs.totalHits.relation, TotalHits.Relation.EQUAL_TO);
+            assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value());
+            assertEquals(context.queryResult().topDocs().topDocs.totalHits.relation(), TotalHits.Relation.EQUAL_TO);
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(3));
         }
         try (TestSearchContext context = createContext(newContextSearcher(reader), q)) {
@@ -764,9 +764,9 @@ public void testDisableTopScoreCollection() throws Exception {
             );
             assertEquals(collectorManager.newCollector().scoreMode(), org.apache.lucene.search.ScoreMode.TOP_DOCS);
             QueryPhase.executeQuery(context);
-            assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value);
+            assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value());
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(3));
-            assertEquals(context.queryResult().topDocs().topDocs.totalHits.relation, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO);
+            assertEquals(context.queryResult().topDocs().topDocs.totalHits.relation(), TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO);
         }
     }
 
@@ -881,8 +881,8 @@ public void testNumericSortOptimization() throws Exception {
             QueryPhase.addCollectorsAndSearch(searchContext);
             assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints());
             assertThat(searchContext.queryResult().topDocs().topDocs.scoreDocs, arrayWithSize(0));
-            assertThat(searchContext.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs));
-            assertThat(searchContext.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(searchContext.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+            assertThat(searchContext.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO));
         }
 
         // 7. Test that sort optimization doesn't break a case where from = 0 and size= 0
@@ -950,8 +950,8 @@ public void testMaxScoreQueryVisitor() {
 
     // assert score docs are in order and their number is as expected
     private static void assertSortResults(TopDocs topDocs, long totalNumDocs, boolean isDoubleSort) {
-        assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation);
-        assertThat(topDocs.totalHits.value, lessThan(totalNumDocs)); // we collected less docs than total number
+        assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation());
+        assertThat(topDocs.totalHits.value(), lessThan(totalNumDocs)); // we collected less docs than total number
         long cur1, cur2;
         long prev1 = Long.MIN_VALUE;
         long prev2 = Long.MIN_VALUE;
@@ -990,7 +990,7 @@ public void testMinScore() throws Exception {
             context.trackTotalHitsUpTo(5);
 
             QueryPhase.addCollectorsAndSearch(context);
-            assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value);
+            assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value());
         }
     }
 
@@ -1136,7 +1136,7 @@ private static ContextIndexSearcher earlyTerminationContextSearcher(IndexReader
         ) {
 
             @Override
-            public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
+            public void search(LeafReaderContextPartition[] partitions, Weight weight, Collector collector) throws IOException {
                 final Collector in = new FilterCollector(collector) {
                     @Override
                     public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
@@ -1153,7 +1153,7 @@ public void collect(int doc) throws IOException {
                         };
                     }
                 };
-                super.search(leaves, weight, in);
+                super.search(partitions, weight, in);
             }
         };
     }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java
index 3bf9514cad547..b417f7adbc8b7 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java
@@ -32,6 +32,7 @@
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
@@ -132,7 +133,7 @@ private void scorerTimeoutTest(int size, CheckedConsumer<LeafReaderContext, IOEx
             try (SearchContext context = createSearchContext(query, size)) {
                 QueryPhase.executeQuery(context);
                 assertFalse(context.queryResult().searchTimedOut());
-                assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value);
+                assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value());
                 assertEquals(size, context.queryResult().topDocs().topDocs.scoreDocs.length);
             }
         }
@@ -142,7 +143,7 @@ private void scorerTimeoutTest(int size, CheckedConsumer<LeafReaderContext, IOEx
                 QueryPhase.executeQuery(context);
                 assertTrue(context.queryResult().searchTimedOut());
                 int firstSegmentMaxDoc = reader.leaves().get(0).reader().maxDoc();
-                assertEquals(Math.min(2048, firstSegmentMaxDoc), context.queryResult().topDocs().topDocs.totalHits.value);
+                assertEquals(Math.min(2048, firstSegmentMaxDoc), context.queryResult().topDocs().topDocs.totalHits.value());
                 assertEquals(Math.min(size, firstSegmentMaxDoc), context.queryResult().topDocs().topDocs.scoreDocs.length);
             }
         }
@@ -159,14 +160,14 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
                     boolean firstSegment = true;
 
                     @Override
-                    public Scorer scorer(LeafReaderContext context) throws IOException {
+                    public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                         if (firstSegment == false && isTimeoutExpected) {
                             shouldTimeout = true;
                         }
                         timeoutTrigger.accept(context);
                         assert shouldTimeout == false : "should have already timed out";
                         firstSegment = false;
-                        return super.scorer(context);
+                        return super.scorerSupplier(context);
                     }
                 };
             }
@@ -180,7 +181,7 @@ public void testBulkScorerTimeout() throws IOException {
             try (SearchContext context = createSearchContext(query, size)) {
                 QueryPhase.executeQuery(context);
                 assertFalse(context.queryResult().searchTimedOut());
-                assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value);
+                assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value());
                 assertEquals(size, context.queryResult().topDocs().topDocs.scoreDocs.length);
             }
         }
@@ -190,7 +191,7 @@ public void testBulkScorerTimeout() throws IOException {
                 QueryPhase.executeQuery(context);
                 assertTrue(context.queryResult().searchTimedOut());
                 int firstSegmentMaxDoc = reader.leaves().get(0).reader().maxDoc();
-                assertEquals(Math.min(2048, firstSegmentMaxDoc), context.queryResult().topDocs().topDocs.totalHits.value);
+                assertEquals(Math.min(2048, firstSegmentMaxDoc), context.queryResult().topDocs().topDocs.totalHits.value());
                 assertEquals(Math.min(size, firstSegmentMaxDoc), context.queryResult().topDocs().topDocs.scoreDocs.length);
             }
         }
@@ -202,33 +203,49 @@ private static TimeoutQuery newMatchAllBulkScorerTimeoutQuery(boolean timeoutExp
             public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) {
                 return new MatchAllWeight(this, boost, scoreMode) {
                     @Override
-                    public BulkScorer bulkScorer(LeafReaderContext context) {
-                        final float score = score();
-                        final int maxDoc = context.reader().maxDoc();
-                        return new BulkScorer() {
+                    public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                        ScorerSupplier inScorerSupplier = super.scorerSupplier(context);
+                        return new ScorerSupplier() {
                             @Override
-                            public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
-                                assert shouldTimeout == false : "should have already timed out";
-                                max = Math.min(max, maxDoc);
-                                ScoreAndDoc scorer = new ScoreAndDoc();
-                                scorer.score = score;
-                                collector.setScorer(scorer);
-                                for (int doc = min; doc < max; ++doc) {
-                                    scorer.doc = doc;
-                                    if (acceptDocs == null || acceptDocs.get(doc)) {
-                                        collector.collect(doc);
-                                    }
-                                }
-                                if (timeoutExpected) {
-                                    // timeout after collecting the first batch of documents from the 1st segment, or the entire 1st segment
-                                    shouldTimeout = true;
-                                }
-                                return max == maxDoc ? DocIdSetIterator.NO_MORE_DOCS : max;
+                            public Scorer get(long leadCost) throws IOException {
+                                return inScorerSupplier.get(leadCost);
                             }
 
                             @Override
                             public long cost() {
-                                return 0;
+                                return inScorerSupplier.cost();
+                            }
+
+                            @Override
+                            public BulkScorer bulkScorer() throws IOException {
+                                final float score = score();
+                                final int maxDoc = context.reader().maxDoc();
+                                return new BulkScorer() {
+                                    @Override
+                                    public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
+                                        assert shouldTimeout == false : "should have already timed out";
+                                        max = Math.min(max, maxDoc);
+                                        Score scorer = new Score();
+                                        scorer.score = score;
+                                        collector.setScorer(scorer);
+                                        for (int doc = min; doc < max; ++doc) {
+                                            if (acceptDocs == null || acceptDocs.get(doc)) {
+                                                collector.collect(doc);
+                                            }
+                                        }
+                                        if (timeoutExpected) {
+                                            // timeout after collecting the first batch of documents from the 1st segment, or the entire 1st
+                                            // segment
+                                            shouldTimeout = true;
+                                        }
+                                        return max == maxDoc ? DocIdSetIterator.NO_MORE_DOCS : max;
+                                    }
+
+                                    @Override
+                                    public long cost() {
+                                        return 0;
+                                    }
+                                };
                             }
                         };
                     }
@@ -258,14 +275,8 @@ private TestSearchContext createSearchContext(Query query, int size) throws IOEx
         return context;
     }
 
-    private static class ScoreAndDoc extends Scorable {
+    private static class Score extends Scorable {
         float score;
-        int doc = -1;
-
-        @Override
-        public int docID() {
-            return doc;
-        }
 
         @Override
         public float score() {
@@ -315,8 +326,9 @@ protected MatchAllWeight(Query query, float score, ScoreMode scoreMode) {
         }
 
         @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            return new ConstantScoreScorer(this, score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc()));
+        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+            Scorer scorer = new ConstantScoreScorer(score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc()));
+            return new DefaultScorerSupplier(scorer);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
index 5862e1bd1329f..6250d1679fda3 100644
--- a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
@@ -196,7 +196,7 @@ public String getWriteableName() {
 
             @Override
             public TransportVersion getMinimalSupportedVersion() {
-                return TransportVersions.RANK_FEATURE_PHASE_ADDED;
+                return TransportVersions.V_8_15_0;
             }
         };
     }
diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
index 4a02e84bbe4f8..209dfdcc16969 100644
--- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -166,7 +167,7 @@ public MappedFieldType getFieldType(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(
                     name,
                     createDefaultIndexAnalyzers(),
-                    idxSettings.getMode().isSyntheticSourceEnabled()
+                    SourceFieldMapper.isSynthetic(idxSettings)
                 );
                 return builder.build(MapperBuilderContext.root(false, false)).fieldType();
             }
@@ -233,7 +234,7 @@ public MappedFieldType getFieldType(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(
                     name,
                     createDefaultIndexAnalyzers(),
-                    idxSettings.getMode().isSyntheticSourceEnabled()
+                    SourceFieldMapper.isSynthetic(idxSettings)
                 );
                 return builder.build(MapperBuilderContext.root(false, false)).fieldType();
             }
diff --git a/server/src/test/java/org/elasticsearch/search/rescore/RescorePhaseTests.java b/server/src/test/java/org/elasticsearch/search/rescore/RescorePhaseTests.java
new file mode 100644
index 0000000000000..5a1c4b789b460
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/rescore/RescorePhaseTests.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.search.rescore;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.elasticsearch.action.search.SearchShardTask;
+import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardTestCase;
+import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext;
+import org.elasticsearch.search.fetch.subphase.FetchFieldsContext;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.FilteredSearchContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.tasks.TaskCancelHelper;
+import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.test.TestSearchContext;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class RescorePhaseTests extends IndexShardTestCase {
+
+    public void testRescorePhaseCancellation() throws IOException {
+        IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+        try (Directory dir = newDirectory()) {
+            try (RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc)) {
+                final int numDocs = scaledRandomIntBetween(100, 200);
+                for (int i = 0; i < numDocs; ++i) {
+                    Document doc = new Document();
+                    w.addDocument(doc);
+                }
+            }
+            try (IndexReader reader = DirectoryReader.open(dir)) {
+                ContextIndexSearcher s = new ContextIndexSearcher(
+                    reader,
+                    IndexSearcher.getDefaultSimilarity(),
+                    IndexSearcher.getDefaultQueryCache(),
+                    new QueryCachingPolicy() {
+                        @Override
+                        public void onUse(Query query) {}
+
+                        @Override
+                        public boolean shouldCache(Query query) {
+                            return false;
+                        }
+                    },
+                    true
+                );
+                IndexShard shard = newShard(true);
+                try (TestSearchContext context = new TestSearchContext(null, shard, s)) {
+                    context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
+                    SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+                    context.setTask(task);
+                    SearchContext wrapped = new FilteredSearchContext(context) {
+                        @Override
+                        public boolean lowLevelCancellation() {
+                            return true;
+                        }
+
+                        @Override
+                        public FetchDocValuesContext docValuesContext() {
+                            return context.docValuesContext();
+                        }
+
+                        @Override
+                        public SearchContext docValuesContext(FetchDocValuesContext docValuesContext) {
+                            return context.docValuesContext(docValuesContext);
+                        }
+
+                        @Override
+                        public FetchFieldsContext fetchFieldsContext() {
+                            return context.fetchFieldsContext();
+                        }
+
+                        @Override
+                        public SearchContext fetchFieldsContext(FetchFieldsContext fetchFieldsContext) {
+                            return context.fetchFieldsContext(fetchFieldsContext);
+                        }
+                    };
+                    try (wrapped) {
+                        Runnable cancellationChecks = RescorePhase.getCancellationChecks(wrapped);
+                        assertNotNull(cancellationChecks);
+                        TaskCancelHelper.cancel(task, "test cancellation");
+                        assertTrue(wrapped.isCancelled());
+                        expectThrows(TaskCancelledException.class, cancellationChecks::run);
+                        QueryRescorer.QueryRescoreContext rescoreContext = new QueryRescorer.QueryRescoreContext(10);
+                        rescoreContext.setQuery(new ParsedQuery(new MatchAllDocsQuery()));
+                        rescoreContext.setCancellationChecker(cancellationChecks);
+                        expectThrows(
+                            TaskCancelledException.class,
+                            () -> new QueryRescorer().rescore(
+                                new TopDocs(
+                                    new TotalHits(10, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
+                                    new ScoreDoc[] { new ScoreDoc(0, 1.0f) }
+                                ),
+                                context.searcher(),
+                                rescoreContext
+                            )
+                        );
+                    }
+                }
+                closeShards(shard);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java
index ca05c57b7d733..e8f88f3297b78 100644
--- a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java
@@ -160,7 +160,7 @@ public void testRankDocsQueryEarlyTerminate() throws IOException {
                     // so worst case is we could end up collecting up to 1 + max(topSize , totalHitsThreshold) + rankDocs.length documents
                     // as we could have already filled the priority queue with non-optimal docs
                     assertThat(
-                        col.totalHits.value,
+                        col.totalHits.value(),
                         lessThanOrEqualTo((long) (1 + Math.max(topSize, totalHitsThreshold) + rankDocs.length))
                     );
                     assertEqualTopDocs(col.scoreDocs, rankDocs);
@@ -177,7 +177,7 @@ public void testRankDocsQueryEarlyTerminate() throws IOException {
                     );
                     var topDocsManager = new TopScoreDocCollectorManager(topSize, null, Integer.MAX_VALUE);
                     var col = searcher.search(q, topDocsManager);
-                    assertThat(col.totalHits.value, equalTo((long) reader.maxDoc()));
+                    assertThat(col.totalHits.value(), equalTo((long) reader.maxDoc()));
                     assertEqualTopDocs(col.scoreDocs, rankDocs);
                 }
 
@@ -192,9 +192,26 @@ public void testRankDocsQueryEarlyTerminate() throws IOException {
                     );
                     var topDocsManager = new TopScoreDocCollectorManager(topSize, null, Integer.MAX_VALUE);
                     var col = searcher.search(q, topDocsManager);
-                    assertThat(col.totalHits.value, equalTo((long) topSize));
+                    assertThat(col.totalHits.value(), equalTo((long) topSize));
                     assertEqualTopDocs(col.scoreDocs, rankDocs);
                 }
+
+                {
+                    // A single rank doc in the last segment
+                    RankDoc[] singleRankDoc = new RankDoc[1];
+                    singleRankDoc[0] = rankDocs[rankDocs.length - 1];
+                    RankDocsQuery q = new RankDocsQuery(
+                        reader,
+                        singleRankDoc,
+                        new Query[] { NumericDocValuesField.newSlowExactQuery("active", 1) },
+                        new String[1],
+                        false
+                    );
+                    var topDocsManager = new TopScoreDocCollectorManager(1, null, 0);
+                    var col = searcher.search(q, topDocsManager);
+                    assertThat(col.totalHits.value(), lessThanOrEqualTo((long) (2 + rankDocs.length)));
+                    assertEqualTopDocs(col.scoreDocs, singleRankDoc);
+                }
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/search/sort/BucketedSortForFloatsTests.java b/server/src/test/java/org/elasticsearch/search/sort/BucketedSortForFloatsTests.java
index e2eab5e7c4436..b02ac9b80b8b9 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/BucketedSortForFloatsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/BucketedSortForFloatsTests.java
@@ -121,18 +121,12 @@ public void testScorer() throws IOException {
     }
 
     private class MockScorable extends Scorable {
-        private int doc;
         private float score;
 
         @Override
         public float score() throws IOException {
             return score;
         }
-
-        @Override
-        public int docID() {
-            return doc;
-        }
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
index 5ee3f4c0cc110..6b53daeb40c4e 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.analysis.core.SimpleAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -797,7 +796,9 @@ public void testParsingContextFromDocument() {
         LuceneDocument document = new LuceneDocument();
 
         KeywordFieldMapper.KeywordFieldType keyword = new KeywordFieldMapper.KeywordFieldType("category");
-        document.add(new KeywordFieldMapper.KeywordField(keyword.name(), new BytesRef("category1"), new FieldType()));
+        document.add(
+            new KeywordFieldMapper.KeywordField(keyword.name(), new BytesRef("category1"), KeywordFieldMapper.Defaults.FIELD_TYPE)
+        );
         // Ignore doc values
         document.add(new SortedSetDocValuesField(keyword.name(), new BytesRef("category1")));
         Set<String> context = mapping.parseContext(document);
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/server/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
index 320e3fce2e832..7791073ef36fa 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
@@ -211,8 +211,8 @@ public void testFrequencyThreshold() throws Exception {
                     )
                 );
                 assertThat(candidateSet.candidates.length, equalTo(1));
-                assertThat(candidateSet.candidates[0].termStats.docFreq, equalTo(numDocs - 1));
-                assertThat(candidateSet.candidates[0].termStats.totalTermFreq, equalTo((long) numDocs - 1));
+                assertThat(candidateSet.candidates[0].termStats.docFreq(), equalTo(numDocs - 1));
+                assertThat(candidateSet.candidates[0].termStats.totalTermFreq(), equalTo((long) numDocs - 1));
 
                 // test that it doesn't overflow
                 assertThat(generator.thresholdTermFrequency(Integer.MAX_VALUE), equalTo(Integer.MAX_VALUE));
@@ -227,8 +227,8 @@ public void testFrequencyThreshold() throws Exception {
                     )
                 );
                 assertThat(candidateSet.candidates.length, equalTo(1));
-                assertThat(candidateSet.candidates[0].termStats.docFreq, equalTo(numDocs - 1));
-                assertThat(candidateSet.candidates[0].termStats.totalTermFreq, equalTo((long) numDocs - 1));
+                assertThat(candidateSet.candidates[0].termStats.docFreq(), equalTo(numDocs - 1));
+                assertThat(candidateSet.candidates[0].termStats.totalTermFreq(), equalTo((long) numDocs - 1));
 
                 // test that it doesn't overflow
                 assertThat(generator.thresholdTermFrequency(Integer.MAX_VALUE), equalTo(Integer.MAX_VALUE));
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractDenseVectorQueryTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractDenseVectorQueryTestCase.java
index dc9d026af0135..72ae45fd26143 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractDenseVectorQueryTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractDenseVectorQueryTestCase.java
@@ -226,7 +226,7 @@ public void testRandom() throws IOException {
                     int n = random().nextInt(100) + 1;
                     TopDocs results = searcher.search(query, n);
                     assert reader.hasDeletions() == false;
-                    assertTrue(results.totalHits.value >= results.scoreDocs.length);
+                    assertTrue(results.totalHits.value() >= results.scoreDocs.length);
                     // verify the results are in descending score order
                     float last = Float.MAX_VALUE;
                     for (ScoreDoc scoreDoc : results.scoreDocs) {
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java
index 18d5c8c85fbec..bef0bbfd27ff6 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java
@@ -282,8 +282,8 @@ public void testScoreDocQuery() throws IOException {
                 final Weight w = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0f);
 
                 TopDocs topDocs = searcher.search(query, 100);
-                assertEquals(scoreDocs.length, topDocs.totalHits.value);
-                assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation);
+                assertEquals(scoreDocs.length, topDocs.totalHits.value());
+                assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation());
 
                 Arrays.sort(topDocs.scoreDocs, Comparator.comparingInt(scoreDoc -> scoreDoc.doc));
                 assertEquals(scoreDocs.length, topDocs.scoreDocs.length);
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java
index d9fe421bafb46..4e4d2158a9574 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java
@@ -74,7 +74,6 @@ public void testParseSearchRequest() throws IOException {
             () -> null,
             () -> null,
             () -> null,
-            () -> null,
             Collections::emptyList,
             () -> null,
             () -> null
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/VectorSimilarityQueryTests.java b/server/src/test/java/org/elasticsearch/search/vectors/VectorSimilarityQueryTests.java
index 8d9fa847a988c..f2ead93ebb6e1 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/VectorSimilarityQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/VectorSimilarityQueryTests.java
@@ -62,14 +62,14 @@ public void testSimpleEuclidean() throws Exception {
                     new VectorSimilarityQuery(new KnnFloatVectorQuery("float_vector", new float[] { 1, 1, 1 }, 5), 3f, 0.25f),
                     5
                 );
-                assertThat(docs.totalHits.value, equalTo(5L));
+                assertThat(docs.totalHits.value(), equalTo(5L));
 
                 // Should match only 4
                 docs = searcher.search(
                     new VectorSimilarityQuery(new KnnFloatVectorQuery("float_vector", new float[] { 1, 1, 1 }, 5), 1f, 0.5f),
                     5
                 );
-                assertThat(docs.totalHits.value, equalTo(4L));
+                assertThat(docs.totalHits.value(), equalTo(4L));
             }
         }
     }
@@ -138,14 +138,14 @@ public void testSimpleCosine() throws IOException {
                     new VectorSimilarityQuery(new KnnFloatVectorQuery("float_vector", new float[] { 1, 1, 1 }, 5), .8f, .9f),
                     5
                 );
-                assertThat(docs.totalHits.value, equalTo(5L));
+                assertThat(docs.totalHits.value(), equalTo(5L));
 
                 // Should match only 4
                 docs = searcher.search(
                     new VectorSimilarityQuery(new KnnFloatVectorQuery("float_vector", new float[] { 1, 1, 1 }, 5), .9f, 0.95f),
                     5
                 );
-                assertThat(docs.totalHits.value, equalTo(4L));
+                assertThat(docs.totalHits.value(), equalTo(4L));
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index f8c3edcbb9d42..c46d98fe1cd8b 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -372,7 +372,7 @@ public void testSuccessfulSnapshotAndRestore() {
 
         final AtomicBoolean documentCountVerified = new AtomicBoolean();
         continueOrDie(searchResponseListener, r -> {
-            assertEquals(documents, Objects.requireNonNull(r.getHits().getTotalHits()).value);
+            assertEquals(documents, Objects.requireNonNull(r.getHits().getTotalHits()).value());
             documentCountVerified.set(true);
         });
 
@@ -816,7 +816,10 @@ public void testConcurrentSnapshotRestoreAndDeleteOther() {
 
         var response = safeResult(searchResponseListener);
         try {
-            assertEquals(documentsFirstSnapshot + documentsSecondSnapshot, Objects.requireNonNull(response.getHits().getTotalHits()).value);
+            assertEquals(
+                documentsFirstSnapshot + documentsSecondSnapshot,
+                Objects.requireNonNull(response.getHits().getTotalHits()).value()
+            );
         } finally {
             response.decRef();
         }
@@ -1177,7 +1180,7 @@ public void testSuccessfulSnapshotWithConcurrentDynamicMappingUpdates() {
         final AtomicBoolean documentCountVerified = new AtomicBoolean();
 
         continueOrDie(searchResponseStepListener, r -> {
-            final long hitCount = r.getHits().getTotalHits().value;
+            final long hitCount = r.getHits().getTotalHits().value();
             assertThat(
                 "Documents were restored but the restored index mapping was older than some documents and misses some of their fields",
                 (int) hitCount,
diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java
index 78d76476d06fc..e5cdecd25ef34 100644
--- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java
+++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java
@@ -157,13 +157,13 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
 
             parentTransportService.addSendBehavior(sendRequestBehavior);
 
-            AbstractSimpleTransportTestCase.connectToNode(parentTransportService, childTransportService.getLocalDiscoNode());
+            AbstractSimpleTransportTestCase.connectToNode(parentTransportService, childTransportService.getLocalNode());
 
             final CancellableTask parentTask = (CancellableTask) parentTransportService.getTaskManager()
                 .register("transport", "internal:testAction", new ParentRequest());
 
             parentTransportService.sendChildRequest(
-                childTransportService.getLocalDiscoNode(),
+                childTransportService.getLocalNode(),
                 "internal:testAction[c]",
                 new EmptyRequest(),
                 parentTask,
@@ -172,7 +172,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
             );
 
             try (MockLog mockLog = MockLog.capture(TaskCancellationService.class)) {
-                for (MockLog.LoggingExpectation expectation : expectations.apply(childTransportService.getLocalDiscoNode())) {
+                for (MockLog.LoggingExpectation expectation : expectations.apply(childTransportService.getLocalNode())) {
                     mockLog.addExpectation(expectation);
                 }
 
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java
index 43dce7d406ba2..d7cf38828b7ba 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java
@@ -89,7 +89,7 @@ public void testRemoteTaskCancellationOnFailedResponse() throws Exception {
         ) {
             remoteTransport.getTaskManager().setTaskCancellationService(new TaskCancellationService(remoteTransport));
             Settings.Builder builder = Settings.builder();
-            builder.putList("cluster.remote.cluster1.seeds", remoteTransport.getLocalDiscoNode().getAddress().toString());
+            builder.putList("cluster.remote.cluster1.seeds", remoteTransport.getLocalNode().getAddress().toString());
             try (
                 MockTransportService localService = MockTransportService.createNewService(
                     builder.build(),
@@ -163,11 +163,11 @@ public void testSearchShards() throws Exception {
             MockTransportService seedTransport = startTransport("seed_node", knownNodes);
             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes)
         ) {
-            knownNodes.add(seedTransport.getLocalDiscoNode());
-            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(seedTransport.getLocalNode());
+            knownNodes.add(discoverableTransport.getLocalNode());
             Collections.shuffle(knownNodes, random());
             Settings.Builder builder = Settings.builder();
-            builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString());
+            builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalNode().getAddress().toString());
             try (
                 MockTransportService service = MockTransportService.createNewService(
                     builder.build(),
@@ -216,11 +216,11 @@ public void testSearchShardsThreadContextHeader() {
             MockTransportService seedTransport = startTransport("seed_node", knownNodes);
             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes)
         ) {
-            knownNodes.add(seedTransport.getLocalDiscoNode());
-            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(seedTransport.getLocalNode());
+            knownNodes.add(discoverableTransport.getLocalNode());
             Collections.shuffle(knownNodes, random());
             Settings.Builder builder = Settings.builder();
-            builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString());
+            builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalNode().getAddress().toString());
             try (
                 MockTransportService service = MockTransportService.createNewService(
                     builder.build(),
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
index ff0742c89bba9..0efaef7015649 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
@@ -73,7 +73,7 @@ public void testConnectAndExecuteRequest() throws Exception {
                 remoteSettings
             )
         ) {
-            DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
+            DiscoveryNode remoteNode = remoteTransport.getLocalNode();
 
             Settings localSettings = Settings.builder()
                 .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE))
@@ -152,7 +152,7 @@ public void testEnsureWeReconnect() throws Exception {
                 remoteSettings
             )
         ) {
-            DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
+            DiscoveryNode remoteNode = remoteTransport.getLocalNode();
             Settings localSettings = Settings.builder()
                 .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE))
                 .put("cluster.remote.test.seeds", remoteNode.getAddress().getAddress() + ":" + remoteNode.getAddress().getPort())
@@ -251,7 +251,7 @@ public void testQuicklySkipUnavailableClusters() throws Exception {
                 remoteSettings
             )
         ) {
-            DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
+            DiscoveryNode remoteNode = remoteTransport.getLocalNode();
 
             Settings localSettings = Settings.builder()
                 .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE))
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index 7a259cf3100f0..21346bb93ef8e 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -311,11 +311,11 @@ public void testCloseWhileConcurrentlyConnecting() throws IOException, Interrupt
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
-            DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode();
-            knownNodes.add(seedTransport.getLocalDiscoNode());
-            knownNodes.add(discoverableTransport.getLocalDiscoNode());
-            knownNodes.add(seedTransport1.getLocalDiscoNode());
+            DiscoveryNode seedNode = seedTransport.getLocalNode();
+            DiscoveryNode seedNode1 = seedTransport1.getLocalNode();
+            knownNodes.add(seedTransport.getLocalNode());
+            knownNodes.add(discoverableTransport.getLocalNode());
+            knownNodes.add(seedTransport1.getLocalNode());
             Collections.shuffle(knownNodes, random());
             List<String> seedNodes = addresses(seedNode1, seedNode);
             Collections.shuffle(seedNodes, random());
@@ -447,9 +447,9 @@ private void doTestGetConnectionInfo(boolean hasClusterCredentials) throws Excep
                 seedTransportSettings
             )
         ) {
-            DiscoveryNode node1 = transport1.getLocalDiscoNode();
-            DiscoveryNode node2 = transport3.getLocalDiscoNode();
-            DiscoveryNode node3 = transport2.getLocalDiscoNode();
+            DiscoveryNode node1 = transport1.getLocalNode();
+            DiscoveryNode node2 = transport3.getLocalNode();
+            DiscoveryNode node3 = transport2.getLocalNode();
             if (hasClusterCredentials) {
                 node1 = node1.withTransportAddress(transport1.boundRemoteAccessAddress().publishAddress());
                 node2 = node2.withTransportAddress(transport3.boundRemoteAccessAddress().publishAddress());
@@ -645,7 +645,7 @@ private void doTestCollectNodes(boolean hasClusterCredentials) throws Exception
                 seedTransportSettings
             )
         ) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode seedNode = seedTransport.getLocalNode();
             if (hasClusterCredentials) {
                 seedNode = seedNode.withTransportAddress(seedTransport.boundRemoteAccessAddress().publishAddress());
             }
@@ -725,8 +725,8 @@ public void testNoChannelsExceptREG() throws Exception {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
-            knownNodes.add(seedTransport.getLocalDiscoNode());
+            DiscoveryNode seedNode = seedTransport.getLocalNode();
+            knownNodes.add(seedTransport.getLocalNode());
             try (
                 MockTransportService service = MockTransportService.createNewService(
                     Settings.EMPTY,
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
index 11243ba088f8f..3633128c45bfa 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
@@ -141,10 +141,10 @@ public void testGroupClusterIndices() throws IOException {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
-            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
-            knownNodes.add(cluster1Transport.getLocalDiscoNode());
-            knownNodes.add(cluster2Transport.getLocalDiscoNode());
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode();
+            knownNodes.add(cluster1Transport.getLocalNode());
+            knownNodes.add(cluster2Transport.getLocalNode());
             Collections.shuffle(knownNodes, random());
 
             try (
@@ -343,10 +343,10 @@ public void testGroupIndices() throws IOException {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
-            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
-            knownNodes.add(cluster1Transport.getLocalDiscoNode());
-            knownNodes.add(cluster2Transport.getLocalDiscoNode());
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode();
+            knownNodes.add(cluster1Transport.getLocalNode());
+            knownNodes.add(cluster2Transport.getLocalNode());
             Collections.shuffle(knownNodes, random());
 
             try (
@@ -453,10 +453,10 @@ public void testIncrementallyAddClusters() throws IOException {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
-            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
-            knownNodes.add(cluster1Transport.getLocalDiscoNode());
-            knownNodes.add(cluster2Transport.getLocalDiscoNode());
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode();
+            knownNodes.add(cluster1Transport.getLocalNode());
+            knownNodes.add(cluster2Transport.getLocalNode());
             Collections.shuffle(knownNodes, random());
 
             try (
@@ -526,8 +526,8 @@ public void testDefaultPingSchedule() throws IOException {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
-            knownNodes.add(seedTransport.getLocalDiscoNode());
+            DiscoveryNode seedNode = seedTransport.getLocalNode();
+            knownNodes.add(seedTransport.getLocalNode());
             TimeValue pingSchedule;
             Settings.Builder settingsBuilder = Settings.builder();
             settingsBuilder.putList("cluster.remote.cluster_1.seeds", seedNode.getAddress().toString());
@@ -582,10 +582,10 @@ public void testCustomPingSchedule() throws IOException {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
-            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
-            knownNodes.add(cluster1Transport.getLocalDiscoNode());
-            knownNodes.add(cluster2Transport.getLocalDiscoNode());
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode();
+            knownNodes.add(cluster1Transport.getLocalNode());
+            knownNodes.add(cluster2Transport.getLocalNode());
             Collections.shuffle(knownNodes, random());
             Settings.Builder settingsBuilder = Settings.builder();
             if (randomBoolean()) {
@@ -635,8 +635,8 @@ public void testChangeSettings() throws Exception {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
-            knownNodes.add(cluster1Transport.getLocalDiscoNode());
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode();
+            knownNodes.add(cluster1Transport.getLocalNode());
             Collections.shuffle(knownNodes, random());
 
             try (
@@ -716,10 +716,10 @@ public void testRemoteNodeAttribute() throws IOException, InterruptedException {
                 gateway
             )
         ) {
-            final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode();
-            final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode();
-            final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode();
-            final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode();
+            final DiscoveryNode c1N1Node = c1N1.getLocalNode();
+            final DiscoveryNode c1N2Node = c1N2.getLocalNode();
+            final DiscoveryNode c2N1Node = c2N1.getLocalNode();
+            final DiscoveryNode c2N2Node = c2N2.getLocalNode();
             knownNodes.add(c1N1Node);
             knownNodes.add(c1N2Node);
             knownNodes.add(c2N1Node);
@@ -809,10 +809,10 @@ public void testRemoteNodeRoles() throws IOException, InterruptedException {
                 data
             )
         ) {
-            final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode();
-            final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode();
-            final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode();
-            final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode();
+            final DiscoveryNode c1N1Node = c1N1.getLocalNode();
+            final DiscoveryNode c1N2Node = c1N2.getLocalNode();
+            final DiscoveryNode c2N1Node = c2N1.getLocalNode();
+            final DiscoveryNode c2N2Node = c2N2.getLocalNode();
             knownNodes.add(c1N1Node);
             knownNodes.add(c1N2Node);
             knownNodes.add(c2N1Node);
@@ -906,10 +906,10 @@ public void testCollectNodes() throws InterruptedException, IOException {
                 settings
             )
         ) {
-            final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode();
-            final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode();
-            final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode();
-            final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode();
+            final DiscoveryNode c1N1Node = c1N1.getLocalNode();
+            final DiscoveryNode c1N2Node = c1N2.getLocalNode();
+            final DiscoveryNode c2N1Node = c2N1.getLocalNode();
+            final DiscoveryNode c2N2Node = c2N2.getLocalNode();
             knownNodes_c1.add(c1N1Node);
             knownNodes_c1.add(c1N2Node);
             knownNodes_c2.add(c2N1Node);
@@ -1170,8 +1170,8 @@ public void testReconnectWhenStrategySettingsUpdated() throws Exception {
             )
         ) {
 
-            final DiscoveryNode node0 = cluster_node_0.getLocalDiscoNode();
-            final DiscoveryNode node1 = cluster_node_1.getLocalDiscoNode();
+            final DiscoveryNode node0 = cluster_node_0.getLocalNode();
+            final DiscoveryNode node1 = cluster_node_1.getLocalNode();
             knownNodes.add(node0);
             knownNodes.add(node1);
             Collections.shuffle(knownNodes, random());
@@ -1267,10 +1267,10 @@ public void testSkipUnavailable() {
                 TransportVersion.current()
             )
         ) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode seedNode = seedTransport.getLocalNode();
             knownNodes.add(seedNode);
             Settings.Builder builder = Settings.builder();
-            builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString());
+            builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalNode().getAddress().toString());
             try (
                 MockTransportService service = MockTransportService.createNewService(
                     builder.build(),
@@ -1353,8 +1353,8 @@ public void testUseDifferentTransportProfileForCredentialsProtectedRemoteCluster
             );
             MockTransportService c2 = startTransport("cluster_2", knownNodes, VersionInformation.CURRENT, TransportVersion.current());
         ) {
-            final DiscoveryNode c1Node = c1.getLocalDiscoNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress());
-            final DiscoveryNode c2Node = c2.getLocalDiscoNode();
+            final DiscoveryNode c1Node = c1.getLocalNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress());
+            final DiscoveryNode c2Node = c2.getLocalNode();
 
             final MockSecureSettings secureSettings = new MockSecureSettings();
             secureSettings.setString("cluster.remote.cluster_1.credentials", randomAlphaOfLength(10));
@@ -1441,7 +1441,7 @@ public void testUpdateRemoteClusterCredentialsRebuildsConnectionWithCorrectProfi
                     .build()
             )
         ) {
-            final DiscoveryNode discoNode = c.getLocalDiscoNode().withTransportAddress(c.boundRemoteAccessAddress().publishAddress());
+            final DiscoveryNode discoNode = c.getLocalNode().withTransportAddress(c.boundRemoteAccessAddress().publishAddress());
             try (
                 MockTransportService transportService = MockTransportService.createNewService(
                     Settings.EMPTY,
@@ -1518,8 +1518,8 @@ public void testUpdateRemoteClusterCredentialsRebuildsMultipleConnectionsDespite
                     .build()
             )
         ) {
-            final DiscoveryNode c1DiscoNode = c1.getLocalDiscoNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress());
-            final DiscoveryNode c2DiscoNode = c2.getLocalDiscoNode().withTransportAddress(c2.boundRemoteAccessAddress().publishAddress());
+            final DiscoveryNode c1DiscoNode = c1.getLocalNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress());
+            final DiscoveryNode c2DiscoNode = c2.getLocalNode().withTransportAddress(c2.boundRemoteAccessAddress().publishAddress());
             try (
                 MockTransportService transportService = MockTransportService.createNewService(
                     Settings.EMPTY,
@@ -1636,7 +1636,7 @@ public void testLogsConnectionResult() throws IOException {
 
             assertThatLogger(
                 () -> clusterSettings.applySettings(
-                    Settings.builder().putList("cluster.remote.remote_1.seeds", remote.getLocalDiscoNode().getAddress().toString()).build()
+                    Settings.builder().putList("cluster.remote.remote_1.seeds", remote.getLocalNode().getAddress().toString()).build()
                 ),
                 RemoteClusterService.class,
                 new MockLog.SeenEventExpectation(
diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java
index 398bb5f2a9106..46585ac382583 100644
--- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java
@@ -85,15 +85,15 @@ public void setUp() throws Exception {
         threadPool = new TestThreadPool(getClass().getName());
         serviceA = buildService(version0, transportVersion0); // this one supports dynamic tracer updates
         serviceA.taskManager.setTaskCancellationService(new TaskCancellationService(serviceA));
-        nodeA = serviceA.getLocalDiscoNode();
+        nodeA = serviceA.getLocalNode();
         serviceB = buildService(version1, transportVersion1); // this one doesn't support dynamic tracer updates
         serviceB.taskManager.setTaskCancellationService(new TaskCancellationService(serviceB));
-        nodeB = serviceB.getLocalDiscoNode();
+        nodeB = serviceB.getLocalNode();
         serviceC = buildService(version1, transportVersion1); // this one doesn't support dynamic tracer updates
         serviceC.taskManager.setTaskCancellationService(new TaskCancellationService(serviceC));
-        nodeC = serviceC.getLocalDiscoNode();
+        nodeC = serviceC.getLocalNode();
         serviceD = buildService(version1, transportVersion1);
-        nodeD = serviceD.getLocalDiscoNode();
+        nodeD = serviceD.getLocalNode();
     }
 
     @Override
diff --git a/settings.gradle b/settings.gradle
index a47751fd499c0..39453e8d0935a 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -17,7 +17,7 @@ pluginManagement {
 }
 
 plugins {
-  id "com.gradle.develocity" version "3.17.4"
+  id "com.gradle.develocity" version "3.18.1"
   id 'elasticsearch.java-toolchain'
 }
 
@@ -63,8 +63,6 @@ List projects = [
   'distribution:archives:linux-aarch64-tar',
   'distribution:archives:linux-tar',
   'distribution:docker',
-  'distribution:docker:cloud-docker-export',
-  'distribution:docker:cloud-docker-aarch64-export',
   'distribution:docker:cloud-ess-docker-export',
   'distribution:docker:cloud-ess-docker-aarch64-export',
   'distribution:docker:docker-aarch64-export',
@@ -75,8 +73,6 @@ List projects = [
   'distribution:docker:ubi-docker-export',
   'distribution:docker:wolfi-docker-aarch64-export',
   'distribution:docker:wolfi-docker-export',
-  'distribution:docker:wolfi-ess-docker-aarch64-export',
-  'distribution:docker:wolfi-ess-docker-export',
   'distribution:packages:aarch64-deb',
   'distribution:packages:deb',
   'distribution:packages:aarch64-rpm',
diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
index 809bcd5a9bc12..008a056e87901 100644
--- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
+++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
@@ -129,6 +129,15 @@ public void testSortByManyLongsTooMuchMemoryAsync() throws IOException {
                 try {
                     resp = client().performRequest(fetch);
                 } catch (ResponseException e) {
+                    if (e.getResponse().getStatusLine().getStatusCode() == 403) {
+                        /*
+                         * There's a bug when loading from the translog with security
+                         * enabled. If we retry a few times we'll load from the index
+                         * itself and should succeed.
+                         */
+                        logger.error("polled for results got 403");
+                        continue;
+                    }
                     if (e.getResponse().getStatusLine().getStatusCode() == 404) {
                         logger.error("polled for results got 404");
                         continue;
diff --git a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java
index 9e5b9dd0be547..7f2cb85919d10 100644
--- a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java
+++ b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java
@@ -140,7 +140,7 @@ public void testRetrieveSnapshots() throws Exception {
 
         logger.info("--> run a search");
         assertResponse(client.prepareSearch("test-idx").setQuery(QueryBuilders.termQuery("text", "sometext")), searchResponse -> {
-            assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+            assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
             assertThat(COUNTS.intValue(), greaterThan(0));
         });
     }
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
index d8716fd987f3e..92ce04b6bea5b 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
@@ -12,6 +12,9 @@
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
@@ -23,10 +26,14 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 
+import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -35,6 +42,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
@@ -47,6 +55,8 @@
  */
 @SuppressForbidden(reason = "Uses a HttpServer to emulate an Azure endpoint")
 public class AzureHttpHandler implements HttpHandler {
+    private static final Logger logger = LogManager.getLogger(AzureHttpHandler.class);
+
     private final Map<String, BytesReference> blobs;
     private final String account;
     private final String container;
@@ -264,7 +274,98 @@ public void handle(final HttpExchange exchange) throws IOException {
                 exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
                 exchange.getResponseBody().write(response);
 
+            } else if (Regex.simpleMatch("POST /" + account + "/" + container + "*restype=container*comp=batch*", request)) {
+                // Blob Batch (https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch)
+                final StringBuilder response = new StringBuilder();
+
+                try (BufferedReader requestReader = new BufferedReader(new InputStreamReader(exchange.getRequestBody()))) {
+                    final String batchBoundary = requestReader.readLine();
+                    final String responseBoundary = "batch_" + UUID.randomUUID();
+
+                    String line;
+                    String contentId = null, requestId = null, toDelete = null;
+                    while ((line = requestReader.readLine()) != null) {
+                        if (batchBoundary.equals(line) || (batchBoundary + "--").equals(line)) {
+                            // Found the end of a single request, process it
+                            if (contentId == null || requestId == null || toDelete == null) {
+                                throw new IllegalStateException(
+                                    "Missing contentId/requestId/toDelete: " + contentId + "/" + requestId + "/" + toDelete
+                                );
+                            }
+
+                            // Process the deletion
+                            if (blobs.remove("/" + account + toDelete) != null) {
+                                final String acceptedPart = Strings.format("""
+                                    --%s
+                                    Content-Type: application/http
+                                    Content-ID: %s
+
+                                    HTTP/1.1 202 Accepted
+                                    x-ms-delete-type-permanent: true
+                                    x-ms-request-id: %s
+                                    x-ms-version: 2018-11-09
+
+                                    """, responseBoundary, contentId, requestId).replaceAll("\n", "\r\n");
+                                response.append(acceptedPart);
+                            } else {
+                                final String notFoundBody = Strings.format(
+                                    """
+                                        <?xml version="1.0" encoding="utf-8"?>
+                                        <Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.
+                                        RequestId:%s
+                                        Time:%s</Message></Error>""",
+                                    requestId,
+                                    DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now(ZoneId.of("UTC")))
+                                );
+                                final String notFoundPart = Strings.format("""
+                                    --%s
+                                    Content-Type: application/http
+                                    Content-ID: %s
+
+                                    HTTP/1.1 404 The specified blob does not exist.
+                                    x-ms-error-code: BlobNotFound
+                                    x-ms-request-id: %s
+                                    x-ms-version: 2018-11-09
+                                    Content-Length: %d
+                                    Content-Type: application/xml
+
+                                    %s
+                                    """, responseBoundary, contentId, requestId, notFoundBody.length(), notFoundBody)
+                                    .replaceAll("\n", "\r\n");
+                                response.append(notFoundPart);
+                            }
+
+                            // Clear the state
+                            toDelete = null;
+                            contentId = null;
+                            requestId = null;
+                        } else if (Regex.simpleMatch("x-ms-client-request-id: *", line)) {
+                            if (requestId != null) {
+                                throw new IllegalStateException("Got multiple request IDs in a single request?");
+                            }
+                            requestId = line.split("\\s")[1];
+                        } else if (Regex.simpleMatch("Content-ID: *", line)) {
+                            if (contentId != null) {
+                                throw new IllegalStateException("Got multiple content IDs in a single request?");
+                            }
+                            contentId = line.split("\\s")[1];
+                        } else if (Regex.simpleMatch("DELETE /" + container + "/*", line)) {
+                            String blobName = RestUtils.decodeComponent(line.split("(\\s|\\?)")[1]);
+                            if (toDelete != null) {
+                                throw new IllegalStateException("Got multiple deletes in a single request?");
+                            }
+                            toDelete = blobName;
+                        }
+                    }
+                    response.append("--").append(responseBoundary).append("--\r\n0\r\n");
+                    // Send the response
+                    exchange.getResponseHeaders().add("Content-Type", "multipart/mixed; boundary=" + responseBoundary);
+                    exchange.sendResponseHeaders(RestStatus.ACCEPTED.getStatus(), response.length());
+                    logger.debug("--> Sending response:\n{}", response);
+                    exchange.getResponseBody().write(response.toString().getBytes(StandardCharsets.UTF_8));
+                }
             } else {
+                logger.warn("--> Unrecognised request received: {}", request);
                 sendError(exchange, RestStatus.BAD_REQUEST);
             }
         } finally {
diff --git a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java
index 59205aa546cd2..3f3e0c0a25578 100644
--- a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java
+++ b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java
@@ -11,20 +11,18 @@
 
 import com.sun.net.httpserver.HttpServer;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.junit.rules.ExternalResource;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.io.UncheckedIOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.security.MessageDigest;
+import java.util.List;
+import java.util.Objects;
 
 /**
  * This fixture is used to simulate a maxmind-provided server for downloading maxmind geoip database files from the
@@ -32,21 +30,17 @@
  */
 public class EnterpriseGeoIpHttpFixture extends ExternalResource {
 
-    private final Path source;
-    private final String[] databaseTypes;
+    private final List<String> maxmindDatabaseTypes;
+    private final List<String> ipinfoDatabaseTypes;
     private HttpServer server;
 
     /*
-     * The values in databaseTypes must be in DatabaseConfiguration.MAXMIND_NAMES, and must be one of the databases copied in the
-     * copyFiles method of thisi class.
+     * The values in maxmindDatabaseTypes must be in DatabaseConfiguration.MAXMIND_NAMES, and the ipinfoDatabaseTypes
+     * must be in DatabaseConfiguration.IPINFO_NAMES.
      */
-    public EnterpriseGeoIpHttpFixture(String... databaseTypes) {
-        this.databaseTypes = databaseTypes;
-        try {
-            this.source = Files.createTempDirectory("source");
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
+    public EnterpriseGeoIpHttpFixture(List<String> maxmindDatabaseTypes, List<String> ipinfoDatabaseTypes) {
+        this.maxmindDatabaseTypes = List.copyOf(maxmindDatabaseTypes);
+        this.ipinfoDatabaseTypes = List.copyOf(ipinfoDatabaseTypes);
     }
 
     public String getAddress() {
@@ -55,7 +49,6 @@ public String getAddress() {
 
     @Override
     protected void before() throws Throwable {
-        copyFiles();
         this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
 
         // for expediency reasons, it is handy to have this test fixture be able to serve the dual purpose of actually stubbing
@@ -64,26 +57,33 @@ protected void before() throws Throwable {
         this.server.createContext("/", exchange -> {
             String response = "[]"; // an empty json array
             exchange.sendResponseHeaders(200, response.length());
-            try (OutputStream os = exchange.getResponseBody()) {
-                os.write(response.getBytes(StandardCharsets.UTF_8));
+            try (OutputStream out = exchange.getResponseBody()) {
+                out.write(response.getBytes(StandardCharsets.UTF_8));
             }
         });
 
         // register the file types for the download fixture
-        for (String databaseType : databaseTypes) {
-            createContextForEnterpriseDatabase(databaseType);
+        for (String databaseType : maxmindDatabaseTypes) {
+            createContextForMaxmindDatabase(databaseType);
+        }
+        for (String databaseType : ipinfoDatabaseTypes) {
+            createContextForIpinfoDatabase(databaseType);
         }
 
         server.start();
     }
 
-    private void createContextForEnterpriseDatabase(String databaseType) {
+    private static InputStream fixtureStream(String name) {
+        return Objects.requireNonNull(GeoIpHttpFixture.class.getResourceAsStream(name));
+    }
+
+    private void createContextForMaxmindDatabase(String databaseType) {
         this.server.createContext("/" + databaseType + "/download", exchange -> {
             exchange.sendResponseHeaders(200, 0);
             if (exchange.getRequestURI().toString().contains("sha256")) {
                 MessageDigest sha256 = MessageDigests.sha256();
-                try (InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/" + databaseType + ".tgz")) {
-                    sha256.update(inputStream.readAllBytes());
+                try (InputStream in = fixtureStream("/geoip-fixture/" + databaseType + ".tgz")) {
+                    sha256.update(in.readAllBytes());
                 }
                 exchange.getResponseBody()
                     .write(
@@ -93,10 +93,33 @@ private void createContextForEnterpriseDatabase(String databaseType) {
                     );
             } else {
                 try (
-                    OutputStream outputStream = exchange.getResponseBody();
-                    InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/" + databaseType + ".tgz")
+                    OutputStream out = exchange.getResponseBody();
+                    InputStream in = fixtureStream("/geoip-fixture/" + databaseType + ".tgz")
+                ) {
+                    in.transferTo(out);
+                }
+            }
+            exchange.getResponseBody().close();
+        });
+    }
+
+    private void createContextForIpinfoDatabase(String databaseType) {
+        this.server.createContext("/free/" + databaseType + ".mmdb", exchange -> {
+            exchange.sendResponseHeaders(200, 0);
+            if (exchange.getRequestURI().toString().contains("checksum")) {
+                MessageDigest md5 = MessageDigests.md5();
+                try (InputStream in = fixtureStream("/ipinfo-fixture/ip_" + databaseType + "_sample.mmdb")) {
+                    md5.update(in.readAllBytes());
+                }
+                exchange.getResponseBody().write(Strings.format("""
+                    { "checksums": { "md5": "%s" } }
+                    """, MessageDigests.toHexString(md5.digest())).getBytes(StandardCharsets.UTF_8));
+            } else {
+                try (
+                    OutputStream out = exchange.getResponseBody();
+                    InputStream in = fixtureStream("/ipinfo-fixture/ip_" + databaseType + "_sample.mmdb")
                 ) {
-                    inputStream.transferTo(outputStream);
+                    in.transferTo(out);
                 }
             }
             exchange.getResponseBody().close();
@@ -107,14 +130,4 @@ private void createContextForEnterpriseDatabase(String databaseType) {
     protected void after() {
         server.stop(0);
     }
-
-    private void copyFiles() throws Exception {
-        for (String databaseType : databaseTypes) {
-            Files.copy(
-                GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoIP2-City.tgz"),
-                source.resolve(databaseType + ".tgz"),
-                StandardCopyOption.REPLACE_EXISTING
-            );
-        }
-    }
 }
diff --git a/test/fixtures/geoip-fixture/src/main/resources/ipinfo-fixture/ip_asn_sample.mmdb b/test/fixtures/geoip-fixture/src/main/resources/ipinfo-fixture/ip_asn_sample.mmdb
new file mode 100644
index 0000000000000..3e1fc49ba48a5
Binary files /dev/null and b/test/fixtures/geoip-fixture/src/main/resources/ipinfo-fixture/ip_asn_sample.mmdb differ
diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index 6a0b4f2dd2c1b..a11291e846ec1 100644
--- a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -151,10 +151,7 @@ public static ClusterState state(
             .settings(indexSettings(IndexVersion.current(), 1, numberOfReplicas).put(SETTING_CREATION_DATE, System.currentTimeMillis()))
             .primaryTerm(0, primaryTerm)
             .timestampRange(timeFieldRange)
-            .eventIngestedRange(
-                timeFieldRange,
-                timeFieldRange == IndexLongFieldRange.UNKNOWN ? null : TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE
-            )
+            .eventIngestedRange(timeFieldRange, timeFieldRange == IndexLongFieldRange.UNKNOWN ? null : TransportVersions.V_8_15_0)
             .build();
 
         IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
@@ -287,10 +284,7 @@ public static ClusterState state(final int numberOfNodes, final String[] indices
                 .settings(
                     indexSettings(IndexVersion.current(), numberOfPrimaries, 0).put(SETTING_CREATION_DATE, System.currentTimeMillis())
                 )
-                .eventIngestedRange(
-                    IndexLongFieldRange.UNKNOWN,
-                    randomFrom(TransportVersions.V_8_0_0, TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)
-                )
+                .eventIngestedRange(IndexLongFieldRange.UNKNOWN, randomFrom(TransportVersions.V_8_0_0, TransportVersions.V_8_15_0))
                 .build();
 
             IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex());
diff --git a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java
index d03396f9b53b3..64b468226e509 100644
--- a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java
+++ b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java
@@ -11,7 +11,6 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
-import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -19,14 +18,12 @@
 import org.elasticsearch.cluster.version.CompatibilityVersionsUtils;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.plugins.MetadataUpgrader;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
-import java.io.IOException;
 import java.util.List;
 
 import static org.mockito.Mockito.mock;
@@ -70,11 +67,6 @@ public void start(Settings settings, NodeEnvironment nodeEnvironment, NamedXCont
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
         );
         final MetaStateService metaStateService = mock(MetaStateService.class);
-        try {
-            when(metaStateService.loadFullState()).thenReturn(new Tuple<>(Manifest.empty(), Metadata.builder().build()));
-        } catch (IOException e) {
-            throw new AssertionError(e);
-        }
         start(
             settings,
             transportService,
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index 0b5803e9887d6..4713adf6cf01d 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -178,7 +178,7 @@ protected static void assertVisibleCount(Engine engine, int numDocs, boolean ref
             engine.refresh("test");
         }
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+            Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()));
             assertThat(totalHits, equalTo(numDocs));
         }
     }
@@ -970,7 +970,7 @@ protected static void assertVisibleCount(InternalEngine engine, int numDocs, boo
             engine.refresh("test");
         }
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-            Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+            Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(searcher.getSlices()));
             assertThat(totalHits, equalTo(numDocs));
         }
     }
@@ -1168,7 +1168,10 @@ public static void assertOpsOnReplica(
         assertVisibleCount(replicaEngine, lastFieldValue == null ? 0 : 1);
         if (lastFieldValue != null) {
             try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
-                Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager());
+                Integer totalHits = searcher.search(
+                    new TermQuery(new Term("value", lastFieldValue)),
+                    new TotalHitCountCollectorManager(searcher.getSlices())
+                );
                 assertThat(totalHits, equalTo(1));
             }
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
index 98f18829966c7..47a227cebc956 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
@@ -502,7 +502,7 @@ private void assertQueryOnlyOnText(String queryName, ThrowingRunnable buildQuery
     }
 
     protected final String readSource(IndexReader reader, int docId) throws IOException {
-        return reader.document(docId).getBinaryValue("_source").utf8ToString();
+        return reader.storedFields().document(docId).getBinaryValue("_source").utf8ToString();
     }
 
     protected final void checkExpensiveQuery(BiConsumer<MappedFieldType, SearchExecutionContext> queryBuilder) {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
index b6aa3f97241e1..1c4cfa4ec7ff9 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
@@ -8,6 +8,7 @@
  */
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.DocValuesSkipIndexType;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -139,6 +140,7 @@ public FieldInfo getFieldInfoWithName(String name) {
             randomBoolean(),
             IndexOptions.NONE,
             DocValuesType.NONE,
+            DocValuesSkipIndexType.NONE,
             -1,
             new HashMap<>(),
             1,
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
index 0d05c3d0cd77b..502ffdde62e5a 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
@@ -37,6 +37,11 @@ public class KeywordFieldSyntheticSourceSupport implements MapperTestCase.Synthe
         this.docValues = useFallbackSyntheticSource == false || ESTestCase.randomBoolean();
     }
 
+    @Override
+    public boolean ignoreAbove() {
+        return ignoreAbove != null;
+    }
+
     @Override
     public boolean preservesExactSource() {
         // We opt in into fallback synthetic source implementation
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index a9ee0317ce1ee..3960aa5a91cc5 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -142,7 +142,7 @@ protected static String randomIndexOptions() {
 
     protected final DocumentMapper createDocumentMapper(XContentBuilder mappings, IndexMode indexMode) throws IOException {
         return switch (indexMode) {
-            case STANDARD -> createDocumentMapper(mappings);
+            case STANDARD, LOOKUP -> createDocumentMapper(mappings);
             case TIME_SERIES -> createTimeSeriesModeDocumentMapper(mappings);
             case LOGSDB -> createLogsModeDocumentMapper(mappings);
         };
@@ -179,6 +179,11 @@ public final MapperService createMapperService(XContentBuilder mappings) throws
         return createMapperService(getVersion(), mappings);
     }
 
+    public final MapperService createSytheticSourceMapperService(XContentBuilder mappings) throws IOException {
+        var settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
+        return createMapperService(getVersion(), settings, () -> true, mappings);
+    }
+
     protected IndexVersion getVersion() {
         return IndexVersion.current();
     }
@@ -302,8 +307,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) {}
                 mapperMetrics
             );
 
-            if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) {
-                mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE);
+            if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping(indexSettings) != null) {
+                mapperService.merge(
+                    null,
+                    indexSettings.getMode().getDefaultMapping(indexSettings),
+                    MapperService.MergeReason.MAPPING_UPDATE
+                );
             }
 
             return mapperService;
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
index d8298d49d3ebc..c89c0b2e37dd2 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.script.ScriptFactory;
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.LeafStoredFieldsLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.Source;
@@ -1103,6 +1104,10 @@ default boolean preservesExactSource() {
             return false;
         }
 
+        default boolean ignoreAbove() {
+            return false;
+        }
+
         /**
          * Examples that should work when source is generated from doc values.
          */
@@ -1321,12 +1326,12 @@ private BlockLoader getBlockLoader(boolean columnReader) {
             return mapper.fieldType(loaderFieldName).blockLoader(new MappedFieldType.BlockLoaderContext() {
                 @Override
                 public String indexName() {
-                    throw new UnsupportedOperationException();
+                    return mapper.getIndexSettings().getIndex().getName();
                 }
 
                 @Override
                 public IndexSettings indexSettings() {
-                    throw new UnsupportedOperationException();
+                    return mapper.getIndexSettings();
                 }
 
                 @Override
@@ -1359,9 +1364,19 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
 
     private void testBlockLoader(boolean syntheticSource, boolean columnReader) throws IOException {
         // TODO if we're not using synthetic source use a different sort of example. Or something.
-        SyntheticSourceExample example = syntheticSourceSupport(false, columnReader).example(5);
+        var syntheticSourceSupport = syntheticSourceSupport(false, columnReader);
+        SyntheticSourceExample example = syntheticSourceSupport.example(5);
+        if (syntheticSource && columnReader == false) {
+            // The synthetic source testing support can't always handle now the difference between stored and synthetic source mode.
+            // In case of ignore above, the ignored values are always appended after the valid values
+            // (both if field has doc values or stored field). While stored source just reads original values (from _source) and there
+            // is no notion of values that are ignored.
+            // TODO: fix this by improving block loader support: https://github.com/elastic/elasticsearch/issues/115257
+            assumeTrue("inconsistent synthetic source testing support with ignore above", syntheticSourceSupport.ignoreAbove() == false);
+        }
+        // TODO: only rely index.mapping.source.mode setting
         XContentBuilder mapping = syntheticSource ? syntheticSourceFieldMapping(example.mapping) : fieldMapping(example.mapping);
-        MapperService mapper = createMapperService(mapping);
+        MapperService mapper = syntheticSource ? createSytheticSourceMapperService(mapping) : createMapperService(mapping);
         BlockReaderSupport blockReaderSupport = getSupportedReaders(mapper, "field");
         if (syntheticSource) {
             // geo_point and point do not yet support synthetic source
@@ -1370,11 +1385,16 @@ private void testBlockLoader(boolean syntheticSource, boolean columnReader) thro
                 blockReaderSupport.syntheticSource
             );
         }
-        testBlockLoader(columnReader, example, blockReaderSupport);
+        var sourceLoader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP);
+        testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader);
     }
 
-    protected final void testBlockLoader(boolean columnReader, SyntheticSourceExample example, BlockReaderSupport blockReaderSupport)
-        throws IOException {
+    protected final void testBlockLoader(
+        boolean columnReader,
+        SyntheticSourceExample example,
+        BlockReaderSupport blockReaderSupport,
+        SourceLoader sourceLoader
+    ) throws IOException {
         BlockLoader loader = blockReaderSupport.getBlockLoader(columnReader);
         Function<Object, Object> valuesConvert = loadBlockExpected(blockReaderSupport, columnReader);
         if (valuesConvert == null) {
@@ -1401,9 +1421,15 @@ protected final void testBlockLoader(boolean columnReader, SyntheticSourceExampl
                         return;
                     }
                 } else {
+                    StoredFieldsSpec storedFieldsSpec = loader.rowStrideStoredFieldSpec();
+                    if (storedFieldsSpec.requiresSource()) {
+                        storedFieldsSpec = storedFieldsSpec.merge(
+                            new StoredFieldsSpec(true, storedFieldsSpec.requiresMetadata(), sourceLoader.requiredStoredFields())
+                        );
+                    }
                     BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader(
-                        StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null),
-                        loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null
+                        StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx, null),
+                        storedFieldsSpec.requiresSource() ? sourceLoader.leaf(ctx.reader(), null) : null
                     );
                     storedFieldsLoader.advanceTo(0);
                     BlockLoader.Builder builder = loader.builder(TestBlock.factory(ctx.reader().numDocs()), 1);
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java
index b6a031c9ff906..97ded7f9a06f2 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java
@@ -51,24 +51,9 @@ public static MapperTestCase.BlockReaderSupport getSupportedReaders(MapperServic
     }
 
     public static Function<Object, Object> loadBlockExpected(MapperTestCase.BlockReaderSupport blockReaderSupport, boolean columnReader) {
-        if (nullLoaderExpected(blockReaderSupport.mapper(), blockReaderSupport.loaderFieldName())) {
-            return null;
-        }
         return v -> ((BytesRef) v).utf8ToString();
     }
 
-    private static boolean nullLoaderExpected(MapperService mapper, String fieldName) {
-        MappedFieldType type = mapper.fieldType(fieldName);
-        if (type instanceof TextFieldMapper.TextFieldType t) {
-            if (t.isSyntheticSource() == false || t.canUseSyntheticSourceDelegateForQuerying() || t.isStored()) {
-                return false;
-            }
-            String parentField = mapper.mappingLookup().parentField(fieldName);
-            return parentField == null || nullLoaderExpected(mapper, parentField);
-        }
-        return false;
-    }
-
     public static void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) {
         // `reader` here is reader of original document and `roundTripReader` reads document
         // created from synthetic source.
@@ -98,6 +83,11 @@ private static class TextFieldFamilySyntheticSourceSupport implements MapperTest
             );
         }
 
+        @Override
+        public boolean ignoreAbove() {
+            return keywordMultiFieldSyntheticSourceSupport.ignoreAbove();
+        }
+
         @Override
         public MapperTestCase.SyntheticSourceExample example(int maxValues) {
             if (store) {
diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java
index 2ddd153b8a936..c76967e5d00ac 100644
--- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -198,7 +198,9 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
         entry("daitchmokotoffsoundex", Void.class),
         entry("persianstem", Void.class),
         // not exposed
-        entry("word2vecsynonym", Void.class)
+        entry("word2vecsynonym", Void.class),
+        // not exposed
+        entry("romaniannormalization", Void.class)
     );
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = Map.of(
diff --git a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
index 568f386a81fd1..ec85feb200984 100644
--- a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
@@ -156,11 +156,11 @@ protected void checkTransientErrorsDuringRecoveryAreRetried(String recoveryActio
         Runnable connectionBreaker = () -> {
             // Always break connection from source to remote to ensure that actions are retried
             logger.info("--> closing connections from source node to target node");
-            blueTransportService.disconnectFromNode(redTransportService.getLocalDiscoNode());
+            blueTransportService.disconnectFromNode(redTransportService.getLocalNode());
             if (randomBoolean()) {
                 // Sometimes break connection from remote to source to ensure that recovery is re-established
                 logger.info("--> closing connections from target node to source node");
-                redTransportService.disconnectFromNode(blueTransportService.getLocalDiscoNode());
+                redTransportService.disconnectFromNode(blueTransportService.getLocalNode());
             }
         };
         TransientReceiveRejected handlingBehavior = new TransientReceiveRejected(recoveryActionToBlock, recoveryStarted, connectionBreaker);
@@ -258,13 +258,13 @@ public void checkDisconnectsWhileRecovering(String recoveryActionToBlock) throws
             blueMockTransportService.addRequestHandlingBehavior(recoveryActionToBlock, (handler, request, channel, task) -> {
                 logger.info("--> preventing {} response by closing response channel", recoveryActionToBlock);
                 requestFailed.countDown();
-                redMockTransportService.disconnectFromNode(blueMockTransportService.getLocalDiscoNode());
+                redMockTransportService.disconnectFromNode(blueMockTransportService.getLocalNode());
                 handler.messageReceived(request, channel, task);
             });
             redMockTransportService.addRequestHandlingBehavior(recoveryActionToBlock, (handler, request, channel, task) -> {
                 logger.info("--> preventing {} response by closing response channel", recoveryActionToBlock);
                 requestFailed.countDown();
-                blueMockTransportService.disconnectFromNode(redMockTransportService.getLocalDiscoNode());
+                blueMockTransportService.disconnectFromNode(redMockTransportService.getLocalNode());
                 handler.messageReceived(request, channel, task);
             });
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java
deleted file mode 100644
index cda571c0ef4b5..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.logsdb.datageneration;
-
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.logsdb.datageneration.fields.TopLevelObjectFieldDataGenerator;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Entry point of data generation logic.
- * Every instance of generator generates a random mapping and a document generation routine
- * that produces randomly generated documents valid for this mapping.
- */
-public class DataGenerator {
-    private final TopLevelObjectFieldDataGenerator topLevelGenerator;
-
-    public DataGenerator(DataGeneratorSpecification specification) {
-        this.topLevelGenerator = new TopLevelObjectFieldDataGenerator(specification);
-    }
-
-    /**
-     * Writes a fully built mapping document (enclosed in a top-level object) to a provided builder.
-     * @param mapping destination
-     * @throws IOException
-     */
-    public void writeMapping(XContentBuilder mapping) throws IOException {
-        mapping.startObject().field("_doc");
-        topLevelGenerator.mappingWriter(Map.of()).accept(mapping);
-        mapping.endObject();
-    }
-
-    /**
-     * Writes a fully built mapping document (enclosed in a top-level object) to a provided builder.
-     * Allows customizing parameters of top level object mapper.
-     * @param mapping destination
-     * @param customMappingParameters writer of custom mapping parameters of top level object mapping
-     * @throws IOException
-     */
-    public void writeMapping(XContentBuilder mapping, Map<String, Object> customMappingParameters) throws IOException {
-        mapping.startObject().field("_doc");
-        topLevelGenerator.mappingWriter(customMappingParameters).accept(mapping);
-        mapping.endObject();
-    }
-
-    /**
-     * Generates a document and writes it to a provided builder. New document is generated every time.
-     * @param document
-     * @throws IOException
-     */
-    public void generateDocument(XContentBuilder document) throws IOException {
-        topLevelGenerator.fieldValueGenerator(b -> {}).accept(document);
-    }
-
-    /**
-     * Generates a document and writes it to a provided builder. New document is generated every time.
-     * Supports appending custom content to generated document (e.g. a custom generated field).
-     * @param document
-     * @param customDocumentModifications
-     * @throws IOException
-     */
-    public void generateDocument(XContentBuilder document, CheckedConsumer<XContentBuilder, IOException> customDocumentModifications)
-        throws IOException {
-        topLevelGenerator.fieldValueGenerator(customDocumentModifications).accept(document);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java
index 97eb8b91a6e64..f6ff7c521afcc 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java
@@ -18,7 +18,7 @@
 import java.util.List;
 
 /**
- * Allows configuring behavior of {@link  DataGenerator}.
+ * Allows configuring behavior of data generation components.
  * @param dataSource source of generated data
  * @param maxFieldCountPerLevel maximum number of fields that an individual object in mapping has.
  *                              Applies to subobjects.
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java
new file mode 100644
index 0000000000000..9b2878ff7bfc8
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
+
+/**
+ * Generator that generates a valid random document that follows the structure of provided {@link Template}.
+ */
+public class DocumentGenerator {
+    private final DataGeneratorSpecification specification;
+
+    private final DataSourceResponse.ObjectArrayGenerator objectArrayGenerator;
+
+    public DocumentGenerator(DataGeneratorSpecification specification) {
+        this.specification = specification;
+
+        this.objectArrayGenerator = specification.dataSource().get(new DataSourceRequest.ObjectArrayGenerator());
+    }
+
+    /**
+     * Generates a valid random document following the provided template.
+     * @param template template for the document
+     * @param mapping generated mapping that will be applied to the destination index of this document
+     * @return document as a map where subobjects are represented as nested maps
+     */
+    public Map<String, Object> generate(Template template, Mapping mapping) {
+        var documentMap = new TreeMap<String, Object>();
+        for (var predefinedField : specification.predefinedFields()) {
+            documentMap.put(predefinedField.name(), predefinedField.generator(specification.dataSource()).generateValue());
+        }
+
+        generateFields(documentMap, template.template(), new Context("", mapping.lookup()));
+        return documentMap;
+    }
+
+    private void generateFields(Map<String, Object> document, Map<String, Template.Entry> template, Context context) {
+        for (var entry : template.entrySet()) {
+            String fieldName = entry.getKey();
+            Template.Entry templateEntry = entry.getValue();
+
+            if (templateEntry instanceof Template.Leaf leaf) {
+                // Unsigned long does not play well when dynamically mapped because
+                // it gets mapped as just long and large values fail to index.
+                // Just skip it.
+                if (leaf.type() == FieldType.UNSIGNED_LONG && context.mappingLookup().get(context.pathTo(fieldName)) == null) {
+                    continue;
+                }
+
+                var generator = leaf.type().generator(fieldName, specification.dataSource());
+
+                document.put(fieldName, generator.generateValue());
+            } else if (templateEntry instanceof Template.Object object) {
+                Optional<Integer> arrayLength = objectArrayGenerator.lengthGenerator().get();
+
+                if (arrayLength.isPresent()) {
+                    var children = new ArrayList<>(arrayLength.get());
+                    document.put(object.name(), children);
+
+                    for (int i = 0; i < arrayLength.get(); i++) {
+                        children.add(generateObject(object, context));
+                    }
+                } else {
+                    document.put(object.name(), generateObject(object, context));
+                }
+            }
+        }
+    }
+
+    private Map<String, Object> generateObject(Template.Object object, Context context) {
+        var children = new TreeMap<String, Object>();
+        generateFields(children, object.children(), context.stepIntoObject(object.name()));
+        return children;
+    }
+
+    record Context(String path, Map<String, Map<String, Object>> mappingLookup) {
+        Context stepIntoObject(String name) {
+            return new Context(pathTo(name), mappingLookup);
+        }
+
+        String pathTo(String leafFieldName) {
+            return path.isEmpty() ? leafFieldName : path + "." + leafFieldName;
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java
index 3d7a96f55cbce..7e28a0a0fab25 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java
@@ -9,21 +9,11 @@
 
 package org.elasticsearch.logsdb.datageneration;
 
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-
 /**
- * Entity responsible for generating a valid randomized mapping for a field
- * and a generator of field values valid for this mapping.
+ * Entity responsible for generating a valid value for a field.
  *
- * Generator is expected to produce the same mapping per instance of generator.
- * Function returned by {@link FieldDataGenerator#fieldValueGenerator() } is expected
- * to produce a randomized value each time.
+ * Generator is expected to produce a different value on every call.
  */
 public interface FieldDataGenerator {
-    CheckedConsumer<XContentBuilder, IOException> mappingWriter();
-
-    CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator();
+    Object generateValue();
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java
index 4304a8c77e60b..96b75f29382e2 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.logsdb.datageneration;
 
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.FloatFieldDataGenerator;
@@ -26,33 +25,40 @@
  * Lists all leaf field types that are supported for data generation.
  */
 public enum FieldType {
-    KEYWORD,
-    LONG,
-    UNSIGNED_LONG,
-    INTEGER,
-    SHORT,
-    BYTE,
-    DOUBLE,
-    FLOAT,
-    HALF_FLOAT,
-    SCALED_FLOAT;
+    KEYWORD("keyword"),
+    LONG("long"),
+    UNSIGNED_LONG("unsigned_long"),
+    INTEGER("integer"),
+    SHORT("short"),
+    BYTE("byte"),
+    DOUBLE("double"),
+    FLOAT("float"),
+    HALF_FLOAT("half_float"),
+    SCALED_FLOAT("scaled_float");
 
-    public FieldDataGenerator generator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    private final String name;
+
+    FieldType(String name) {
+        this.name = name;
+    }
+
+    public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
         return switch (this) {
-            case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case LONG -> new LongFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case SHORT -> new ShortFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
-            case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator);
+            case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource);
+            case LONG -> new LongFieldDataGenerator(fieldName, dataSource);
+            case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource);
+            case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource);
+            case SHORT -> new ShortFieldDataGenerator(fieldName, dataSource);
+            case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource);
+            case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource);
+            case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource);
+            case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource);
+            case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource);
         };
     }
+
+    @Override
+    public String toString() {
+        return name;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Mapping.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Mapping.java
new file mode 100644
index 0000000000000..c20c99d7cb0ba
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Mapping.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import java.util.Map;
+
+/**
+ * Contains generated mapping and supporting data.
+ * @param raw mapping represented as a possibly nested map (maps represent (sub-)objects)
+ * @param lookup supporting data structure that represent mapping in a flat form (full path to field -> mapping parameters)
+ */
+public record Mapping(Map<String, Object> raw, Map<String, Map<String, Object>> lookup) {}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java
new file mode 100644
index 0000000000000..bdb80ee09868f
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Generator that generates a valid random mapping that follows the structure of provided {@link Template}.
+ * Mapping will contain all fields from the template with generated mapping parameters.
+ */
+public class MappingGenerator {
+    private final DataGeneratorSpecification specification;
+
+    private final DataSourceResponse.DynamicMappingGenerator dynamicMappingGenerator;
+
+    public MappingGenerator(DataGeneratorSpecification specification) {
+        this.specification = specification;
+
+        this.dynamicMappingGenerator = specification.dataSource().get(new DataSourceRequest.DynamicMappingGenerator());
+    }
+
+    /**
+     * Generates a valid random mapping following the provided template.
+     * @param template template for the mapping
+     * @return {@link Mapping}
+     */
+    public Mapping generate(Template template) {
+        var lookup = new TreeMap<String, Map<String, Object>>();
+
+        // Top level mapping parameters
+        var mappingParametersGenerator = specification.dataSource()
+            .get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false, ObjectMapper.Subobjects.ENABLED))
+            .mappingGenerator();
+
+        var topLevelMappingParameters = mappingParametersGenerator.get();
+        // Top-level object can't be disabled because @timestamp is a required field in data streams.
+        topLevelMappingParameters.remove("enabled");
+
+        var rawMapping = new TreeMap<String, Object>();
+
+        var childrenMapping = new TreeMap<String, Object>();
+        for (var predefinedField : specification.predefinedFields()) {
+            if (predefinedField.mapping() != null) {
+                childrenMapping.put(predefinedField.name(), predefinedField.mapping());
+                lookup.put(predefinedField.name(), predefinedField.mapping());
+            }
+        }
+        topLevelMappingParameters.put("properties", childrenMapping);
+
+        rawMapping.put("_doc", topLevelMappingParameters);
+
+        if (specification.fullyDynamicMapping()) {
+            // Has to be "true" for fully dynamic mapping
+            topLevelMappingParameters.remove("dynamic");
+
+            return new Mapping(rawMapping, lookup);
+        }
+
+        var dynamicMapping = topLevelMappingParameters.getOrDefault("dynamic", "true").equals("strict")
+            ? DynamicMapping.FORBIDDEN
+            : DynamicMapping.SUPPORTED;
+        var subobjects = ObjectMapper.Subobjects.from(topLevelMappingParameters.getOrDefault("subobjects", "true"));
+
+        generateMapping(childrenMapping, lookup, template.template(), new Context(new HashSet<>(), "", subobjects, dynamicMapping));
+
+        return new Mapping(rawMapping, lookup);
+    }
+
+    private void generateMapping(
+        Map<String, Object> mapping,
+        Map<String, Map<String, Object>> lookup,
+        Map<String, Template.Entry> template,
+        Context context
+    ) {
+        for (var entry : template.entrySet()) {
+            String fieldName = entry.getKey();
+            Template.Entry templateEntry = entry.getValue();
+
+            var mappingParameters = new TreeMap<String, Object>();
+
+            boolean isDynamicallyMapped = isDynamicallyMapped(templateEntry, context);
+            // Simply skip this field if it is dynamic.
+            // Lookup will contain null signaling dynamic mapping as well.
+            if (isDynamicallyMapped) {
+                continue;
+            }
+
+            if (templateEntry instanceof Template.Leaf leaf) {
+                // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic.
+                if (leaf.type() == FieldType.KEYWORD) {
+                    context.addCopyToCandidate(fieldName);
+                }
+
+                var mappingParametersGenerator = specification.dataSource()
+                    .get(
+                        new DataSourceRequest.LeafMappingParametersGenerator(
+                            fieldName,
+                            leaf.type(),
+                            context.eligibleCopyToDestinations(),
+                            context.parentDynamicMapping()
+                        )
+                    )
+                    .mappingGenerator();
+
+                mappingParameters.put("type", leaf.type().toString());
+                mappingParameters.putAll(mappingParametersGenerator.get());
+
+            } else if (templateEntry instanceof Template.Object object) {
+                var mappingParametersGenerator = specification.dataSource()
+                    .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, object.nested(), context.parentSubobjects()))
+                    .mappingGenerator();
+
+                mappingParameters.put("type", object.nested() ? "nested" : "object");
+                mappingParameters.putAll(mappingParametersGenerator.get());
+
+                var childrenMapping = new TreeMap<String, Object>();
+                mappingParameters.put("properties", childrenMapping);
+                generateMapping(
+                    childrenMapping,
+                    lookup,
+                    object.children(),
+                    context.stepIntoObject(object.name(), object.nested(), mappingParameters)
+                );
+            }
+
+            mapping.put(fieldName, mappingParameters);
+            lookup.put(context.pathTo(fieldName), Map.copyOf(mappingParameters));
+        }
+    }
+
+    private boolean isDynamicallyMapped(Template.Entry templateEntry, Context context) {
+        return context.parentDynamicMapping != DynamicMapping.FORBIDDEN
+            && dynamicMappingGenerator.generator().apply(templateEntry instanceof Template.Object);
+    }
+
+    record Context(
+        Set<String> eligibleCopyToDestinations,
+        String path,
+        ObjectMapper.Subobjects parentSubobjects,
+        DynamicMapping parentDynamicMapping
+    ) {
+        Context stepIntoObject(String name, boolean nested, Map<String, Object> mappingParameters) {
+            var subobjects = determineSubobjects(mappingParameters);
+            var dynamicMapping = determineDynamicMapping(mappingParameters);
+
+            // copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document.
+            return new Context(nested ? new HashSet<>() : eligibleCopyToDestinations, pathTo(name), subobjects, dynamicMapping);
+        }
+
+        void addCopyToCandidate(String leafFieldName) {
+            eligibleCopyToDestinations.add(pathTo(leafFieldName));
+        }
+
+        String pathTo(String leafFieldName) {
+            return path.isEmpty() ? leafFieldName : path + "." + leafFieldName;
+        }
+
+        private DynamicMapping determineDynamicMapping(Map<String, Object> mappingParameters) {
+            if (parentDynamicMapping == DynamicMapping.FORCED) {
+                return DynamicMapping.FORCED;
+            }
+
+            var dynamicParameter = mappingParameters.get("dynamic");
+            // Inherited from parent
+            if (dynamicParameter == null) {
+                return parentDynamicMapping;
+            }
+
+            return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED;
+        }
+
+        private ObjectMapper.Subobjects determineSubobjects(Map<String, Object> mappingParameters) {
+            if (parentSubobjects == ObjectMapper.Subobjects.DISABLED) {
+                return ObjectMapper.Subobjects.DISABLED;
+            }
+
+            return ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true"));
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java
new file mode 100644
index 0000000000000..9bb2d34958a27
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import java.util.Map;
+
+/**
+ * A template used to generate mapping and documents for a test.
+ * Template encodes object structure, names of objects/fields and type of leaf fields.
+ * Having such a template allow to create interchangeable random mappings with different parameters
+ * but the same structure in order to f.e. test introduction of a new parameter.
+ * @param template actual template data
+ */
+public record Template(Map<String, Entry> template) {
+    sealed interface Entry permits Leaf, Object {}
+
+    record Leaf(String name, FieldType type) implements Entry {}
+
+    record Object(String name, boolean nested, Map<String, Entry> children) implements Entry {}
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java
new file mode 100644
index 0000000000000..f0a3a866a2673
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+public class TemplateGenerator {
+    private final DataGeneratorSpecification specification;
+    private final DataSourceResponse.ChildFieldGenerator childFieldGenerator;
+    private final Supplier<DataSourceResponse.FieldTypeGenerator.FieldTypeInfo> fieldTypeGenerator;
+
+    public TemplateGenerator(DataGeneratorSpecification specification) {
+        this.specification = specification;
+        this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification));
+        this.fieldTypeGenerator = specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator()).generator();
+    }
+
+    public Template generate() {
+        var map = new HashMap<String, Template.Entry>();
+
+        generateChildFields(map, 0, new AtomicInteger(0));
+        return new Template(map);
+    }
+
+    private void generateChildFields(Map<String, Template.Entry> mapping, int depth, AtomicInteger nestedFieldsCount) {
+        var existingFieldNames = new HashSet<String>();
+        // no child fields is legal
+        var childFieldsCount = childFieldGenerator.generateChildFieldCount();
+
+        for (int i = 0; i < childFieldsCount; i++) {
+            var fieldName = generateFieldName(existingFieldNames);
+
+            if (depth < specification.maxObjectDepth() && childFieldGenerator.generateRegularSubObject()) {
+                var children = new HashMap<String, Template.Entry>();
+                mapping.put(fieldName, new Template.Object(fieldName, false, children));
+                generateChildFields(children, depth + 1, nestedFieldsCount);
+            } else if (depth <= specification.maxObjectDepth()
+                && nestedFieldsCount.get() < specification.nestedFieldsLimit()
+                && childFieldGenerator.generateNestedSubObject()) {
+                    nestedFieldsCount.incrementAndGet();
+
+                    var children = new HashMap<String, Template.Entry>();
+                    mapping.put(fieldName, new Template.Object(fieldName, true, children));
+                    generateChildFields(children, depth + 1, nestedFieldsCount);
+                } else {
+                    var fieldTypeInfo = fieldTypeGenerator.get();
+                    mapping.put(fieldName, new Template.Leaf(fieldName, fieldTypeInfo.fieldType()));
+                }
+        }
+    }
+
+    private String generateFieldName(Set<String> existingFields) {
+        var fieldName = childFieldGenerator.generateFieldName();
+        while (existingFields.contains(fieldName)) {
+            fieldName = childFieldGenerator.generateFieldName();
+        }
+        existingFields.add(fieldName);
+
+        return fieldName;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java
index 9a9c610e091e2..df28282fca407 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java
@@ -73,4 +73,8 @@ default DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReque
     default DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
         return null;
     }
+
+    default DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) {
+        return null;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java
index 8dee5876aa207..d77925f097b5a 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java
@@ -93,7 +93,7 @@ public DataSourceResponse.ChildFieldGenerator accept(DataSourceHandler handler)
         }
     }
 
-    record FieldTypeGenerator(DynamicMapping dynamicMapping) implements DataSourceRequest<DataSourceResponse.FieldTypeGenerator> {
+    record FieldTypeGenerator() implements DataSourceRequest<DataSourceResponse.FieldTypeGenerator> {
         public DataSourceResponse.FieldTypeGenerator accept(DataSourceHandler handler) {
             return handler.handle(this);
         }
@@ -123,4 +123,10 @@ public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHand
             return handler.handle(this);
         }
     }
+
+    record DynamicMappingGenerator() implements DataSourceRequest<DataSourceResponse.DynamicMappingGenerator> {
+        public DataSourceResponse.DynamicMappingGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java
index a7653c10bbf43..fa8f56b3e071b 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java
@@ -52,7 +52,7 @@ interface ChildFieldGenerator extends DataSourceResponse {
     }
 
     record FieldTypeGenerator(Supplier<FieldTypeInfo> generator) implements DataSourceResponse {
-        public record FieldTypeInfo(FieldType fieldType, boolean dynamic) {}
+        public record FieldTypeInfo(FieldType fieldType) {}
     }
 
     record ObjectArrayGenerator(Supplier<Optional<Integer>> lengthGenerator) implements DataSourceResponse {}
@@ -60,4 +60,6 @@ record ObjectArrayGenerator(Supplier<Optional<Integer>> lengthGenerator) impleme
     record LeafMappingParametersGenerator(Supplier<Map<String, Object>> mappingGenerator) implements DataSourceResponse {}
 
     record ObjectMappingParametersGenerator(Supplier<Map<String, Object>> mappingGenerator) implements DataSourceResponse {}
+
+    record DynamicMappingGenerator(Function<Boolean, Boolean> generator) implements DataSourceResponse {}
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java
index 62c4ef87e8a15..56ec676e53d55 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java
@@ -14,7 +14,6 @@
 
 import java.util.Optional;
 import java.util.Set;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween;
 import static org.elasticsearch.test.ESTestCase.randomDouble;
@@ -26,24 +25,25 @@ public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFiel
         return new DataSourceResponse.ChildFieldGenerator() {
             @Override
             public int generateChildFieldCount() {
+                // no child fields is legal
                 return ESTestCase.randomIntBetween(0, request.specification().maxFieldCountPerLevel());
             }
 
             @Override
             public boolean generateDynamicSubObject() {
-                // Using a static 5% change, this is just a chosen value that can be tweaked.
+                // Using a static 5% chance, this is just a chosen value that can be tweaked.
                 return randomDouble() <= 0.05;
             }
 
             @Override
             public boolean generateNestedSubObject() {
-                // Using a static 5% change, this is just a chosen value that can be tweaked.
+                // Using a static 5% chance, this is just a chosen value that can be tweaked.
                 return randomDouble() <= 0.05;
             }
 
             @Override
             public boolean generateRegularSubObject() {
-                // Using a static 5% change, this is just a chosen value that can be tweaked.
+                // Using a static 5% chance, this is just a chosen value that can be tweaked.
                 return randomDouble() <= 0.05;
             }
 
@@ -60,21 +60,10 @@ public String generateFieldName() {
 
     @Override
     public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
-        Supplier<DataSourceResponse.FieldTypeGenerator.FieldTypeInfo> generator = switch (request.dynamicMapping()) {
-            case FORBIDDEN -> () -> generateFieldTypeInfo(false);
-            case FORCED -> () -> generateFieldTypeInfo(true);
-            case SUPPORTED -> () -> generateFieldTypeInfo(ESTestCase.randomBoolean());
-        };
-
-        return new DataSourceResponse.FieldTypeGenerator(generator);
-    }
-
-    private static DataSourceResponse.FieldTypeGenerator.FieldTypeInfo generateFieldTypeInfo(boolean isDynamic) {
-        var excluded = isDynamic ? EXCLUDED_FROM_DYNAMIC_MAPPING : Set.of();
 
-        var fieldType = ESTestCase.randomValueOtherThanMany(excluded::contains, () -> ESTestCase.randomFrom(FieldType.values()));
-
-        return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(fieldType, isDynamic);
+        return new DataSourceResponse.FieldTypeGenerator(
+            () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(FieldType.values()))
+        );
     }
 
     @Override
@@ -87,4 +76,12 @@ public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectAr
             return Optional.empty();
         });
     }
+
+    @Override
+    public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) {
+        // Using a static 5% chance for objects, this is just a chosen value that can be tweaked.
+        return new DataSourceResponse.DynamicMappingGenerator(
+            isObject -> isObject ? ESTestCase.randomDouble() <= 0.05 : ESTestCase.randomBoolean()
+        );
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java
deleted file mode 100644
index c1ec15a3479b3..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.logsdb.datageneration.fields;
-
-import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-class Context {
-    private final DataGeneratorSpecification specification;
-
-    private final DataSourceResponse.ChildFieldGenerator childFieldGenerator;
-    private final DataSourceResponse.ObjectArrayGenerator objectArrayGenerator;
-
-    private final String path;
-    private final int objectDepth;
-    // We don't need atomicity, but we need to pass counter by reference to accumulate total value from sub-objects.
-    private final AtomicInteger nestedFieldsCount;
-    private final Set<String> eligibleCopyToDestinations;
-    private final DynamicMapping parentDynamicMapping;
-    private final ObjectMapper.Subobjects currentSubobjectsConfig;
-
-    Context(
-        DataGeneratorSpecification specification,
-        DynamicMapping parentDynamicMapping,
-        ObjectMapper.Subobjects currentSubobjectsConfig
-    ) {
-        this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping, currentSubobjectsConfig);
-    }
-
-    private Context(
-        DataGeneratorSpecification specification,
-        String path,
-        int objectDepth,
-        AtomicInteger nestedFieldsCount,
-        Set<String> eligibleCopyToDestinations,
-        DynamicMapping parentDynamicMapping,
-        ObjectMapper.Subobjects currentSubobjectsConfig
-    ) {
-        this.specification = specification;
-        this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification));
-        this.objectArrayGenerator = specification.dataSource().get(new DataSourceRequest.ObjectArrayGenerator());
-        this.path = path;
-        this.objectDepth = objectDepth;
-        this.nestedFieldsCount = nestedFieldsCount;
-        this.eligibleCopyToDestinations = eligibleCopyToDestinations;
-        this.parentDynamicMapping = parentDynamicMapping;
-        this.currentSubobjectsConfig = currentSubobjectsConfig;
-    }
-
-    public DataGeneratorSpecification specification() {
-        return specification;
-    }
-
-    public DataSourceResponse.ChildFieldGenerator childFieldGenerator() {
-        return childFieldGenerator;
-    }
-
-    public DataSourceResponse.FieldTypeGenerator fieldTypeGenerator(DynamicMapping dynamicMapping) {
-        return specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator(dynamicMapping));
-    }
-
-    public Context subObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) {
-        return new Context(
-            specification,
-            pathToField(name),
-            objectDepth + 1,
-            nestedFieldsCount,
-            eligibleCopyToDestinations,
-            dynamicMapping,
-            subobjects
-        );
-    }
-
-    public Context nestedObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) {
-        nestedFieldsCount.incrementAndGet();
-        // copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document.
-        return new Context(
-            specification,
-            pathToField(name),
-            objectDepth + 1,
-            nestedFieldsCount,
-            new HashSet<>(),
-            dynamicMapping,
-            subobjects
-        );
-    }
-
-    public boolean shouldAddDynamicObjectField(DynamicMapping dynamicMapping) {
-        if (objectDepth >= specification.maxObjectDepth() || dynamicMapping == DynamicMapping.FORBIDDEN) {
-            return false;
-        }
-
-        return childFieldGenerator.generateDynamicSubObject();
-    }
-
-    public boolean shouldAddObjectField() {
-        if (objectDepth >= specification.maxObjectDepth() || parentDynamicMapping == DynamicMapping.FORCED) {
-            return false;
-        }
-
-        return childFieldGenerator.generateRegularSubObject();
-    }
-
-    public boolean shouldAddNestedField(ObjectMapper.Subobjects subobjects) {
-        if (objectDepth >= specification.maxObjectDepth()
-            || nestedFieldsCount.get() >= specification.nestedFieldsLimit()
-            || parentDynamicMapping == DynamicMapping.FORCED
-            || subobjects == ObjectMapper.Subobjects.DISABLED) {
-            return false;
-        }
-
-        return childFieldGenerator.generateNestedSubObject();
-    }
-
-    public Optional<Integer> generateObjectArray() {
-        if (objectDepth == 0) {
-            return Optional.empty();
-        }
-
-        return objectArrayGenerator.lengthGenerator().get();
-    }
-
-    public DynamicMapping determineDynamicMapping(Map<String, Object> mappingParameters) {
-        if (parentDynamicMapping == DynamicMapping.FORCED) {
-            return DynamicMapping.FORCED;
-        }
-
-        var dynamicParameter = mappingParameters.get("dynamic");
-        // Inherited from parent
-        if (dynamicParameter == null) {
-            return parentDynamicMapping;
-        }
-
-        return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED;
-    }
-
-    public ObjectMapper.Subobjects determineSubobjects(Map<String, Object> mappingParameters) {
-        if (currentSubobjectsConfig == ObjectMapper.Subobjects.DISABLED) {
-            return ObjectMapper.Subobjects.DISABLED;
-        }
-
-        return ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true"));
-    }
-
-    public Set<String> getEligibleCopyToDestinations() {
-        return eligibleCopyToDestinations;
-    }
-
-    public void markFieldAsEligibleForCopyTo(String field) {
-        eligibleCopyToDestinations.add(pathToField(field));
-    }
-
-    private String pathToField(String field) {
-        return path.isEmpty() ? field : path + "." + field;
-    }
-
-    public ObjectMapper.Subobjects getCurrentSubobjectsConfig() {
-        return currentSubobjectsConfig;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java
deleted file mode 100644
index 83a68519d5de1..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.logsdb.datageneration.fields;
-
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
-import org.elasticsearch.logsdb.datageneration.FieldType;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Generic generator for any type of object field (e.g. "object", "nested").
- */
-public class GenericSubObjectFieldDataGenerator {
-    private final Context context;
-
-    GenericSubObjectFieldDataGenerator(Context context) {
-        this.context = context;
-    }
-
-    List<ChildField> generateChildFields(DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) {
-        var existingFieldNames = new HashSet<String>();
-        // no child fields is legal
-        var childFieldsCount = context.childFieldGenerator().generateChildFieldCount();
-        var result = new ArrayList<ChildField>(childFieldsCount);
-
-        for (int i = 0; i < childFieldsCount; i++) {
-            var fieldName = generateFieldName(existingFieldNames);
-
-            if (context.shouldAddDynamicObjectField(dynamicMapping)) {
-                result.add(
-                    new ChildField(
-                        fieldName,
-                        new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED, subobjects)),
-                        true
-                    )
-                );
-            } else if (context.shouldAddObjectField()) {
-                result.add(
-                    new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping, subobjects)), false)
-                );
-            } else if (context.shouldAddNestedField(subobjects)) {
-                result.add(
-                    new ChildField(
-                        fieldName,
-                        new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping, subobjects)),
-                        false
-                    )
-                );
-            } else {
-                var fieldTypeInfo = context.fieldTypeGenerator(dynamicMapping).generator().get();
-
-                // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic.
-                if (fieldTypeInfo.fieldType() == FieldType.KEYWORD) {
-                    context.markFieldAsEligibleForCopyTo(fieldName);
-                }
-
-                var mappingParametersGenerator = context.specification()
-                    .dataSource()
-                    .get(
-                        new DataSourceRequest.LeafMappingParametersGenerator(
-                            fieldName,
-                            fieldTypeInfo.fieldType(),
-                            context.getEligibleCopyToDestinations(),
-                            dynamicMapping
-                        )
-                    );
-                var generator = fieldTypeInfo.fieldType()
-                    .generator(fieldName, context.specification().dataSource(), mappingParametersGenerator);
-                result.add(new ChildField(fieldName, generator, fieldTypeInfo.dynamic()));
-            }
-        }
-
-        return result;
-    }
-
-    List<ChildField> generateChildFields(List<PredefinedField> predefinedFields) {
-        return predefinedFields.stream()
-            .map(pf -> new ChildField(pf.name(), pf.generator(context.specification().dataSource()), false))
-            .toList();
-    }
-
-    static void writeChildFieldsMapping(XContentBuilder mapping, List<ChildField> childFields) throws IOException {
-        for (var childField : childFields) {
-            if (childField.dynamic() == false) {
-                mapping.field(childField.fieldName);
-                childField.generator.mappingWriter().accept(mapping);
-            }
-        }
-    }
-
-    static void writeObjectsData(XContentBuilder document, Context context, CheckedConsumer<XContentBuilder, IOException> objectWriter)
-        throws IOException {
-        var optionalLength = context.generateObjectArray();
-        if (optionalLength.isPresent()) {
-            int size = optionalLength.get();
-
-            document.startArray();
-            for (int i = 0; i < size; i++) {
-                objectWriter.accept(document);
-            }
-            document.endArray();
-        } else {
-            objectWriter.accept(document);
-        }
-    }
-
-    static void writeSingleObject(XContentBuilder document, Iterable<ChildField> childFields) throws IOException {
-        document.startObject();
-        writeChildFieldsData(document, childFields);
-        document.endObject();
-    }
-
-    static void writeChildFieldsData(XContentBuilder document, Iterable<ChildField> childFields) throws IOException {
-        for (var childField : childFields) {
-            document.field(childField.fieldName);
-            childField.generator.fieldValueGenerator().accept(document);
-        }
-    }
-
-    private String generateFieldName(Set<String> existingFields) {
-        var fieldName = context.childFieldGenerator().generateFieldName();
-        while (existingFields.contains(fieldName)) {
-            fieldName = context.childFieldGenerator().generateFieldName();
-        }
-        existingFields.add(fieldName);
-
-        return fieldName;
-    }
-
-    record ChildField(String fieldName, FieldDataGenerator generator, boolean dynamic) {}
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java
deleted file mode 100644
index 69853debf9b77..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.logsdb.datageneration.fields;
-
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-public class NestedFieldDataGenerator implements FieldDataGenerator {
-    private final Context context;
-    private final Map<String, Object> mappingParameters;
-    private final List<GenericSubObjectFieldDataGenerator.ChildField> childFields;
-
-    NestedFieldDataGenerator(Context context) {
-        this.context = context;
-
-        this.mappingParameters = context.specification()
-            .dataSource()
-            .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true, context.getCurrentSubobjectsConfig()))
-            .mappingGenerator()
-            .get();
-        var dynamicMapping = context.determineDynamicMapping(mappingParameters);
-        var subobjects = context.determineSubobjects(mappingParameters);
-
-        var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
-        this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects);
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject();
-
-            b.field("type", "nested");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.startObject("properties");
-            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields);
-            b.endObject();
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        CheckedConsumer<XContentBuilder, IOException> objectWriter = object -> GenericSubObjectFieldDataGenerator.writeSingleObject(
-            object,
-            childFields
-        );
-        return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java
deleted file mode 100644
index 701642c57619b..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.logsdb.datageneration.fields;
-
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-public class ObjectFieldDataGenerator implements FieldDataGenerator {
-    private final Context context;
-    private final Map<String, Object> mappingParameters;
-    private final List<GenericSubObjectFieldDataGenerator.ChildField> childFields;
-
-    ObjectFieldDataGenerator(Context context) {
-        this.context = context;
-
-        this.mappingParameters = context.specification()
-            .dataSource()
-            .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false, context.getCurrentSubobjectsConfig()))
-            .mappingGenerator()
-            .get();
-        var dynamicMapping = context.determineDynamicMapping(mappingParameters);
-        var subobjects = context.determineSubobjects(mappingParameters);
-
-        var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
-        this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects);
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject();
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.startObject("properties");
-            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields);
-            b.endObject();
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        CheckedConsumer<XContentBuilder, IOException> objectWriter = object -> GenericSubObjectFieldDataGenerator.writeSingleObject(
-            object,
-            childFields
-        );
-        return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java
index 57e3ce3ce2a86..948f1cf80b35c 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java
@@ -12,37 +12,21 @@
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.FieldType;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
-import java.util.Set;
+import java.util.Map;
 
 public interface PredefinedField {
     String name();
 
-    FieldDataGenerator generator(DataSource dataSource);
+    FieldType fieldType();
 
-    record WithType(String fieldName, FieldType fieldType, DynamicMapping dynamicMapping) implements PredefinedField {
-        @Override
-        public String name() {
-            return fieldName;
-        }
+    Map<String, Object> mapping();
 
-        @Override
-        public FieldDataGenerator generator(DataSource dataSource) {
-            // copy_to currently not supported for predefined fields, use WithGenerator if needed
-            var mappingParametersGenerator = dataSource.get(
-                new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of(), dynamicMapping)
-            );
-            return fieldType().generator(fieldName, dataSource, mappingParametersGenerator);
-        }
-    }
-
-    record WithGenerator(String fieldName, FieldDataGenerator generator) implements PredefinedField {
-        @Override
-        public String name() {
-            return fieldName;
-        }
+    FieldDataGenerator generator(DataSource dataSource);
 
+    record WithGenerator(String name, FieldType fieldType, Map<String, Object> mapping, FieldDataGenerator generator)
+        implements
+            PredefinedField {
         @Override
         public FieldDataGenerator generator(DataSource dataSource) {
             return generator;
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java
deleted file mode 100644
index 1374362df7f4a..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.logsdb.datageneration.fields;
-
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class TopLevelObjectFieldDataGenerator {
-    private final Context context;
-    private final Map<String, Object> mappingParameters;
-
-    // Child fields of top level object that were explicitly requested, they have predefined name and type.
-    private final List<GenericSubObjectFieldDataGenerator.ChildField> predefinedFields;
-    // Child fields of top level object that are generated and merged with predefined fields.
-    private final List<GenericSubObjectFieldDataGenerator.ChildField> generatedChildFields;
-
-    public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification) {
-        DynamicMapping dynamicMapping;
-        if (specification.fullyDynamicMapping()) {
-            dynamicMapping = DynamicMapping.FORCED;
-            this.mappingParameters = Map.of();
-        } else {
-            this.mappingParameters = new HashMap<>(
-                // Value of subobjects here is for a parent of this object.
-                // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level.
-                specification.dataSource()
-                    .get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false, ObjectMapper.Subobjects.ENABLED))
-                    .mappingGenerator()
-                    .get()
-            );
-            // Top-level object can't be disabled because @timestamp is a required field in data streams.
-            this.mappingParameters.remove("enabled");
-
-            dynamicMapping = mappingParameters.getOrDefault("dynamic", "true").equals("strict")
-                ? DynamicMapping.FORBIDDEN
-                : DynamicMapping.SUPPORTED;
-        }
-        var subobjects = ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true"));
-
-        // Value of subobjects here is for a parent of this object.
-        // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level.
-        this.context = new Context(specification, dynamicMapping, ObjectMapper.Subobjects.ENABLED);
-        var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
-
-        this.predefinedFields = genericGenerator.generateChildFields(specification.predefinedFields());
-        this.generatedChildFields = genericGenerator.generateChildFields(dynamicMapping, subobjects);
-    }
-
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter(Map<String, Object> customMappingParameters) {
-        return b -> {
-            b.startObject();
-
-            var mergedParameters = Stream.of(this.mappingParameters, customMappingParameters)
-                .flatMap(map -> map.entrySet().stream())
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (l, r) -> r));
-
-            for (var entry : mergedParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.startObject("properties");
-            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, predefinedFields);
-            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, generatedChildFields);
-            b.endObject();
-
-            b.endObject();
-        };
-    }
-
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator(
-        CheckedConsumer<XContentBuilder, IOException> customDocumentModification
-    ) {
-        CheckedConsumer<XContentBuilder, IOException> objectWriter = b -> {
-            b.startObject();
-
-            customDocumentModification.accept(b);
-            GenericSubObjectFieldDataGenerator.writeChildFieldsData(b, predefinedFields);
-            GenericSubObjectFieldDataGenerator.writeChildFieldsData(b, generatedChildFields);
-
-            b.endObject();
-        };
-        return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java
index 08cbfa32bcdb2..4ead8ffd0b718 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class ByteFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public ByteFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public ByteFieldDataGenerator(String fieldName, DataSource dataSource) {
         var bytes = dataSource.get(new DataSourceRequest.ByteGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> bytes.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "byte");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java
index 0c486360d0003..cf2c4f6abdbf4 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class DoubleFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public DoubleFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public DoubleFieldDataGenerator(String fieldName, DataSource dataSource) {
         var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "double");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java
index 97397013dd303..b59d5ceabb188 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class FloatFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public FloatFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public FloatFieldDataGenerator(String fieldName, DataSource dataSource) {
         var floats = dataSource.get(new DataSourceRequest.FloatGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> floats.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "float");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java
index 7a8da5fd93117..e2ed299f1a4dc 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class HalfFloatFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public HalfFloatFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public HalfFloatFieldDataGenerator(String fieldName, DataSource dataSource) {
         var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> halfFloats.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "half_float");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java
index 9a942fca3ba14..f2fe8ed8362e5 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class IntegerFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public IntegerFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public IntegerFieldDataGenerator(String fieldName, DataSource dataSource) {
         var ints = dataSource.get(new DataSourceRequest.IntegerGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> ints.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "integer");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java
index 8a37c73d00397..8dc4d8b8767c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class KeywordFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public KeywordFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
         var strings = dataSource.get(new DataSourceRequest.StringGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> strings.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "keyword");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java
index 1c240fd10a953..f17610e501ed7 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class LongFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public LongFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public LongFieldDataGenerator(String fieldName, DataSource dataSource) {
         var longs = dataSource.get(new DataSourceRequest.LongGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> longs.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "long");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java
index 359a725aed941..008dd04179dcd 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class ScaledFloatFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public ScaledFloatFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public ScaledFloatFieldDataGenerator(String fieldName, DataSource dataSource) {
         var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "scaled_float");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java
index 8a056f00701d6..85bff2c85e538 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class ShortFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public ShortFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public ShortFieldDataGenerator(String fieldName, DataSource dataSource) {
         var shorts = dataSource.get(new DataSourceRequest.ShortGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> shorts.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "short");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java
index d836d5625a89c..329f684bef70d 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java
@@ -9,49 +9,25 @@
 
 package org.elasticsearch.logsdb.datageneration.fields.leaf;
 
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
-import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.xcontent.XContentBuilder;
 
-import java.io.IOException;
-import java.util.Map;
 import java.util.function.Supplier;
 
 public class UnsignedLongFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
-    private final Map<String, Object> mappingParameters;
 
-    public UnsignedLongFieldDataGenerator(
-        String fieldName,
-        DataSource dataSource,
-        DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator
-    ) {
+    public UnsignedLongFieldDataGenerator(String fieldName, DataSource dataSource) {
         var unsignedLongs = dataSource.get(new DataSourceRequest.UnsignedLongGenerator());
         var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
         var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
 
         this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> unsignedLongs.generator().get());
-        this.mappingParameters = mappingParametersGenerator.mappingGenerator().get();
     }
 
     @Override
-    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
-        return b -> {
-            b.startObject().field("type", "unsigned_long");
-
-            for (var entry : mappingParameters.entrySet()) {
-                b.field(entry.getKey(), entry.getValue());
-            }
-
-            b.endObject();
-        };
-    }
-
-    @Override
-    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
-        return b -> b.value(valueGenerator.get());
+    public Object generateValue() {
+        return valueGenerator.get();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java
index 90c621c62c305..12094b31a049d 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java
@@ -418,7 +418,9 @@ protected int sendIncompleteContent(HttpExchange exchange, byte[] bytes) throws
         if (bytesToSend > 0) {
             exchange.getResponseBody().write(bytes, rangeStart, bytesToSend);
         }
-        if (randomBoolean()) {
+        if (randomBoolean() || Runtime.version().feature() >= 23) {
+            // For now in JDK23 we need to always flush. See https://bugs.openjdk.org/browse/JDK-8331847.
+            // TODO: remove the JDK version check once that issue is fixed
             exchange.getResponseBody().flush();
         }
         return bytesToSend;
diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
index b59f1a5e5f029..363d34ca3ff86 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
@@ -23,13 +23,11 @@
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.builder.SubSearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.rank.RankBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -122,7 +120,6 @@ public static SearchRequest randomSearchRequest(Supplier<SearchSourceBuilder> ra
     public static SearchSourceBuilder randomSearchSourceBuilder(
         Supplier<HighlightBuilder> randomHighlightBuilder,
         Supplier<SuggestBuilder> randomSuggestBuilder,
-        Supplier<RankBuilder> rankContextBuilderSupplier,
         Supplier<RescorerBuilder<?>> randomRescoreBuilder,
         Supplier<List<SearchExtBuilder>> randomExtBuilders,
         Supplier<CollapseBuilder> randomCollapseBuilder,
@@ -250,17 +247,6 @@ public static SearchSourceBuilder randomSearchSourceBuilder(
         }
         if (randomBoolean()) {
             builder.query(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
-        } else if (randomBoolean()) {
-            builder.subSearches(
-                List.of(
-                    new SubSearchSourceBuilder(
-                        QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20))
-                    ),
-                    new SubSearchSourceBuilder(
-                        QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20))
-                    )
-                )
-            );
         }
         if (randomBoolean()) {
             builder.postFilter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
@@ -354,9 +340,6 @@ public static SearchSourceBuilder randomSearchSourceBuilder(
         if (randomBoolean()) {
             builder.suggest(randomSuggestBuilder.get());
         }
-        if (randomBoolean()) {
-            builder.rankBuilder(rankContextBuilderSupplier.get());
-        }
         if (randomBoolean()) {
             int numRescores = randomIntBetween(1, 5);
             for (int i = 0; i < numRescores; i++) {
diff --git a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java
index cc4aac686a02d..df1ea6b756405 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java
@@ -100,7 +100,7 @@ public static TotalHits getTotalHits(SearchRequestBuilder request) {
     }
 
     public static long getTotalHitsValue(SearchRequestBuilder request) {
-        return getTotalHits(request).value;
+        return getTotalHits(request).value();
     }
 
     public static SearchResponse responseAsSearchResponse(Response searchResponse) throws IOException {
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index d35d5282238ee..5f64d123c1bed 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
 import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.CollectorManager;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -158,6 +159,7 @@
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -831,11 +833,8 @@ protected <R extends InternalAggregation> void debugTestCase(
         QueryCachingPolicy queryCachingPolicy,
         MappedFieldType... fieldTypes
     ) throws IOException {
-        // Don't use searchAndReduce because we only want a single aggregator.
-        IndexSearcher searcher = newIndexSearcher(
-            reader,
-            aggregationBuilder.supportsParallelCollection(field -> getCardinality(reader, field))
-        );
+        // Don't use searchAndReduce because we only want a single aggregator, disable parallel collection too.
+        IndexSearcher searcher = newIndexSearcher(reader, false);
         if (queryCachingPolicy != null) {
             searcher.setQueryCachingPolicy(queryCachingPolicy);
         }
@@ -854,7 +853,21 @@ protected <R extends InternalAggregation> void debugTestCase(
         try {
             Aggregator aggregator = createAggregator(builder, context);
             aggregator.preCollection();
-            searcher.search(context.query(), aggregator.asCollector());
+            searcher.search(context.query(), new CollectorManager<Collector, Void>() {
+                boolean called = false;
+
+                @Override
+                public Collector newCollector() {
+                    assert called == false : "newCollector called multiple times";
+                    called = true;
+                    return aggregator.asCollector();
+                }
+
+                @Override
+                public Void reduce(Collection<Collector> collectors) {
+                    return null;
+                }
+            });
             InternalAggregation r = aggregator.buildTopLevel();
             r = doReduce(
                 List.of(r),
@@ -959,11 +972,11 @@ protected DirectoryReader wrapDirectoryReader(DirectoryReader reader) throws IOE
     }
 
     private static class ShardSearcher extends IndexSearcher {
-        private final List<LeafReaderContext> ctx;
+        private final LeafReaderContextPartition[] ctx;
 
         ShardSearcher(LeafReaderContext ctx, IndexReaderContext parent) {
             super(parent);
-            this.ctx = Collections.singletonList(ctx);
+            this.ctx = new LeafReaderContextPartition[] { IndexSearcher.LeafReaderContextPartition.createForEntireSegment(ctx) };
         }
 
         public void search(Weight weight, Collector collector) throws IOException {
@@ -972,7 +985,7 @@ public void search(Weight weight, Collector collector) throws IOException {
 
         @Override
         public String toString() {
-            return "ShardSearcher(" + ctx.get(0) + ")";
+            return "ShardSearcher(" + ctx[0] + ")";
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
index b1b75c1790287..29112b4bd8f5f 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
@@ -232,7 +232,7 @@ public void setupSuiteScopeCluster() throws Exception {
                 .setSize(5000),
             response -> {
                 assertNoFailures(response);
-                long totalHits = response.getHits().getTotalHits().value;
+                long totalHits = response.getHits().getTotalHits().value();
                 XContentBuilder builder = XContentFactory.jsonBuilder();
                 ChunkedToXContent.wrapAsToXContent(response).toXContent(builder, ToXContent.EMPTY_PARAMS);
                 logger.info("Full high_card_idx Response Content:\n{ {} }", Strings.toString(builder));
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/CentroidAggregationTestBase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/CentroidAggregationTestBase.java
index f191012fb4ef8..f87a87c5ddbc8 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/CentroidAggregationTestBase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/CentroidAggregationTestBase.java
@@ -40,7 +40,7 @@ public void testEmptyAggregation() {
                 .addAggregation(centroidAgg(aggName()).field(SINGLE_VALUED_FIELD_NAME)),
             response -> {
                 CentroidAggregation geoCentroid = response.getAggregations().get(aggName());
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
                 assertThat(geoCentroid, notNullValue());
                 assertThat(geoCentroid.getName(), equalTo(aggName()));
                 assertThat(geoCentroid.centroid(), equalTo(null));
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/SpatialBoundsAggregationTestBase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/SpatialBoundsAggregationTestBase.java
index cb6a58ed65a02..a0fdb0bfabf98 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/SpatialBoundsAggregationTestBase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/SpatialBoundsAggregationTestBase.java
@@ -155,7 +155,7 @@ public void testEmptyAggregation() throws Exception {
         assertNoFailuresAndResponse(
             client().prepareSearch(EMPTY_IDX_NAME).setQuery(matchAllQuery()).addAggregation(boundsAgg(aggName(), SINGLE_VALUED_FIELD_NAME)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
                 SpatialBounds<T> geoBounds = response.getAggregations().get(aggName());
                 assertThat(geoBounds, notNullValue());
                 assertThat(geoBounds.getName(), equalTo(aggName()));
diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java
index 71956b431d9b7..fd41213dcd81d 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/geo/BasePointShapeQueryTestCase.java
@@ -104,7 +104,7 @@ public void testIndexPointsFilterRectangle() throws Exception {
             client().prepareSearch(defaultIndexName)
                 .setQuery(queryBuilder().shapeQuery(defaultFieldName, geometry).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
             }
@@ -113,7 +113,7 @@ public void testIndexPointsFilterRectangle() throws Exception {
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName).setQuery(queryBuilder().shapeQuery(defaultFieldName, geometry)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
             }
@@ -170,7 +170,7 @@ public void testIndexPointsPolygon() throws Exception {
                 .setQuery(queryBuilder().shapeQuery(defaultFieldName, polygon).relation(ShapeRelation.INTERSECTS)),
             response -> {
                 SearchHits searchHits = response.getHits();
-                assertThat(searchHits.getTotalHits().value, equalTo(1L));
+                assertThat(searchHits.getTotalHits().value(), equalTo(1L));
                 assertThat(searchHits.getAt(0).getId(), equalTo("1"));
             }
         );
@@ -209,7 +209,7 @@ public void testIndexPointsMultiPolygon() throws Exception {
             client().prepareSearch(defaultIndexName)
                 .setQuery(queryBuilder().shapeQuery(defaultFieldName, multiPolygon).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 assertThat(response.getHits().getAt(0).getId(), not(equalTo("2")));
                 assertThat(response.getHits().getAt(1).getId(), not(equalTo("2")));
@@ -219,7 +219,7 @@ public void testIndexPointsMultiPolygon() throws Exception {
             client().prepareSearch(defaultIndexName)
                 .setQuery(queryBuilder().shapeQuery(defaultFieldName, multiPolygon).relation(ShapeRelation.WITHIN)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 assertThat(response.getHits().getAt(0).getId(), not(equalTo("2")));
                 assertThat(response.getHits().getAt(1).getId(), not(equalTo("2")));
@@ -229,7 +229,7 @@ public void testIndexPointsMultiPolygon() throws Exception {
             client().prepareSearch(defaultIndexName)
                 .setQuery(queryBuilder().shapeQuery(defaultFieldName, multiPolygon).relation(ShapeRelation.DISJOINT)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
             }
@@ -238,7 +238,7 @@ public void testIndexPointsMultiPolygon() throws Exception {
             client().prepareSearch(defaultIndexName)
                 .setQuery(queryBuilder().shapeQuery(defaultFieldName, multiPolygon).relation(ShapeRelation.CONTAINS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(0L));
                 assertThat(response.getHits().getHits().length, equalTo(0));
             }
         );
@@ -264,7 +264,7 @@ public void testIndexPointsRectangle() throws Exception {
             client().prepareSearch(defaultIndexName)
                 .setQuery(queryBuilder().shapeQuery(defaultFieldName, rectangle).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
             }
@@ -319,7 +319,7 @@ public void testIndexPointsIndexedRectangle() throws Exception {
                         .indexedShapePath(indexedShapePath)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("point2"));
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java
index 91c7a25682ae0..6dca91170c7a5 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java
@@ -206,7 +206,7 @@ public void testRandomGeoCollectionQuery() throws Exception {
 
         QueryBuilder intersects = queryBuilder().intersectionQuery(defaultFieldName, queryCollection);
         assertNoFailuresAndResponse(client().prepareSearch(defaultIndexName).setQuery(intersects), response -> {
-            assertTrue("query: " + intersects + " doc: " + Strings.toString(docSource), response.getHits().getTotalHits().value > 0);
+            assertTrue("query: " + intersects + " doc: " + Strings.toString(docSource), response.getHits().getTotalHits().value() > 0);
         });
     }
 
@@ -352,7 +352,7 @@ public void testEdgeCases() throws Exception {
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName).setQuery(queryBuilder().intersectionQuery(defaultFieldName, query)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("blakely"));
             }
@@ -457,7 +457,7 @@ public void testIndexedShapeReference() throws Exception {
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName).setQuery(queryBuilder().intersectionQuery(defaultFieldName, "Big_Rectangle")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
             }
@@ -466,7 +466,7 @@ public void testIndexedShapeReference() throws Exception {
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName).setQuery(queryBuilder().shapeQuery(defaultFieldName, "Big_Rectangle")),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java
index 4e7378d3a9606..3cd52124d8556 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/geo/DatelinePointShapeQueryTestCase.java
@@ -68,7 +68,7 @@ public void testRectangleSpanningDateline(BasePointShapeQueryTestCase<GeoShapeQu
         GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery(defaultFieldName, rectangle);
         assertResponse(tests.client().prepareSearch(defaultIndexName).setQuery(geoShapeQueryBuilder), response -> {
             SearchHits searchHits = response.getHits();
-            assertEquals(2, searchHits.getTotalHits().value);
+            assertEquals(2, searchHits.getTotalHits().value());
             assertNotEquals("1", searchHits.getAt(0).getId());
             assertNotEquals("1", searchHits.getAt(1).getId());
         });
@@ -112,7 +112,7 @@ public void testPolygonSpanningDateline(BasePointShapeQueryTestCase<GeoShapeQuer
         geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS);
         assertResponse(tests.client().prepareSearch(defaultIndexName).setQuery(geoShapeQueryBuilder), response -> {
             SearchHits searchHits = response.getHits();
-            assertEquals(2, searchHits.getTotalHits().value);
+            assertEquals(2, searchHits.getTotalHits().value());
             assertNotEquals("1", searchHits.getAt(0).getId());
             assertNotEquals("4", searchHits.getAt(0).getId());
             assertNotEquals("1", searchHits.getAt(1).getId());
@@ -155,7 +155,7 @@ public void testMultiPolygonSpanningDateline(BasePointShapeQueryTestCase<GeoShap
         geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS);
         assertResponse(tests.client().prepareSearch(defaultIndexName).setQuery(geoShapeQueryBuilder), response -> {
             SearchHits searchHits = response.getHits();
-            assertEquals(2, searchHits.getTotalHits().value);
+            assertEquals(2, searchHits.getTotalHits().value());
             assertNotEquals("3", searchHits.getAt(0).getId());
             assertNotEquals("3", searchHits.getAt(1).getId());
         });
diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryIntegTestCase.java
index c84d8612b1d4b..97e21f64e2648 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryIntegTestCase.java
@@ -102,7 +102,7 @@ public void testSimpleBoundingBoxTest() throws Exception {
             client().prepareSearch() // from NY
                 .setQuery(geoBoundingBoxQuery("location").setCorners(40.73, -74.1, 40.717, -73.99)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 for (SearchHit hit : response.getHits()) {
                     assertThat(hit.getId(), anyOf(equalTo("1"), equalTo("3"), equalTo("5")));
@@ -114,7 +114,7 @@ public void testSimpleBoundingBoxTest() throws Exception {
             client().prepareSearch() // from NY
                 .setQuery(geoBoundingBoxQuery("location").setCorners(40.73, -74.1, 40.717, -73.99)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 for (SearchHit hit : response.getHits()) {
                     assertThat(hit.getId(), anyOf(equalTo("1"), equalTo("3"), equalTo("5")));
@@ -126,7 +126,7 @@ public void testSimpleBoundingBoxTest() throws Exception {
             client().prepareSearch() // top == bottom && left == right
                 .setQuery(geoBoundingBoxQuery("location").setCorners(40.7143528, -74.0059731, 40.7143528, -74.0059731)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 for (SearchHit hit : response.getHits()) {
                     assertThat(hit.getId(), equalTo("1"));
@@ -138,7 +138,7 @@ public void testSimpleBoundingBoxTest() throws Exception {
             client().prepareSearch() // top == bottom
                 .setQuery(geoBoundingBoxQuery("location").setCorners(40.759011, -74.00009, 40.759011, -73.0059731)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 for (SearchHit hit : response.getHits()) {
                     assertThat(hit.getId(), equalTo("2"));
@@ -150,7 +150,7 @@ public void testSimpleBoundingBoxTest() throws Exception {
             client().prepareSearch() // left == right
                 .setQuery(geoBoundingBoxQuery("location").setCorners(41.8, -73.9844722, 40.7, -73.9844722)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 for (SearchHit hit : response.getHits()) {
                     assertThat(hit.getId(), equalTo("2"));
@@ -163,7 +163,7 @@ public void testSimpleBoundingBoxTest() throws Exception {
             client().prepareSearch() // from NY
                 .setQuery(geoDistanceQuery("location").point(40.5, -73.9).distance(25, DistanceUnit.KILOMETERS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 for (SearchHit hit : response.getHits()) {
                     assertThat(hit.getId(), anyOf(equalTo("7"), equalTo("4")));
diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoShapeQueryTestCase.java
index 4e47b0c51177c..bb57cb132daa2 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoShapeQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoShapeQueryTestCase.java
@@ -143,7 +143,7 @@ public void testEnvelopeSpanningDateline() throws Exception {
             }
         );
         assertResponse(client().prepareSearch(defaultIndexName).setQuery(querySupplier.get()), response -> {
-            assertEquals(2, response.getHits().getTotalHits().value);
+            assertEquals(2, response.getHits().getTotalHits().value());
             assertNotEquals("1", response.getHits().getAt(0).getId());
             assertNotEquals("1", response.getHits().getAt(1).getId());
         });
diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 8bc81fef2157d..7a72a7bd0daf0 100644
--- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.plugins.Plugin;
@@ -366,15 +365,9 @@ protected static Settings.Builder indexSettingsNoReplicas(int shards) {
     /**
      * Randomly write an empty snapshot of an older version to an empty repository to simulate an older repository metadata format.
      */
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    // This used to pick an index version from 7.0.0 to 8.9.0. The minimum now is 8.0.0 but it's not clear what the upper range should be
     protected void maybeInitWithOldSnapshotVersion(String repoName, Path repoPath) throws Exception {
         if (randomBoolean() && randomBoolean()) {
-            initWithSnapshotVersion(
-                repoName,
-                repoPath,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.V_8_9_0)
-            );
+            initWithSnapshotVersion(repoName, repoPath, IndexVersionUtils.randomVersion());
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index 77ff194e2681d..0543bc7a78f8b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -342,6 +342,15 @@ protected static CoordinatorRewriteContext createCoordinatorRewriteContext(
         return serviceHolder.createCoordinatorContext(dateFieldType, min, max);
     }
 
+    protected static CoordinatorRewriteContext createCoordinatorRewriteContext(
+        DateFieldMapper.DateFieldType dateFieldType,
+        long min,
+        long max,
+        String tier
+    ) {
+        return serviceHolder.createCoordinatorContext(dateFieldType, min, max, tier);
+    }
+
     protected static DataRewriteContext dataRewriteContext() {
         return serviceHolder.createDataContext();
     }
@@ -625,13 +634,17 @@ QueryRewriteContext createQueryRewriteContext() {
         }
 
         CoordinatorRewriteContext createCoordinatorContext(DateFieldMapper.DateFieldType dateFieldType, long min, long max) {
+            return createCoordinatorContext(dateFieldType, min, max, "");
+        }
+
+        CoordinatorRewriteContext createCoordinatorContext(DateFieldMapper.DateFieldType dateFieldType, long min, long max, String tier) {
             DateFieldRangeInfo timestampFieldInfo = new DateFieldRangeInfo(
                 dateFieldType,
                 IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(min, max)),
                 dateFieldType,
                 IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(min, max))
             );
-            return new CoordinatorRewriteContext(parserConfiguration, this.client, () -> nowInMillis, timestampFieldInfo);
+            return new CoordinatorRewriteContext(parserConfiguration, this.client, () -> nowInMillis, timestampFieldInfo, tier);
         }
 
         DataRewriteContext createDataContext() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java
index f1ff3492426aa..a93f3b7eaf109 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java
@@ -73,7 +73,7 @@ public static void corruptFile(Random random, Path... files) throws IOException
             long checksumAfterCorruption;
             long actualChecksumAfterCorruption;
 
-            try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
+            try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString())) {
                 assertThat(input.getFilePointer(), is(0L));
                 input.seek(input.length() - CodecUtil.footerLength());
                 checksumAfterCorruption = input.getChecksum();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 87a834d6424b7..d7c5c598ce978 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -908,9 +908,11 @@ public void waitNoPendingTasksOnAll() throws Exception {
     /** Ensures the result counts are as expected, and logs the results if different */
     public void assertResultsAndLogOnFailure(long expectedResults, SearchResponse searchResponse) {
         final TotalHits totalHits = searchResponse.getHits().getTotalHits();
-        if (totalHits.value != expectedResults || totalHits.relation != TotalHits.Relation.EQUAL_TO) {
+        if (totalHits.value() != expectedResults || totalHits.relation() != TotalHits.Relation.EQUAL_TO) {
             StringBuilder sb = new StringBuilder("search result contains [");
-            String value = Long.toString(totalHits.value) + (totalHits.relation == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO ? "+" : "");
+            String value = Long.toString(totalHits.value()) + (totalHits.relation() == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO
+                ? "+"
+                : "");
             sb.append(value).append("] results. expected [").append(expectedResults).append("]");
             String failMsg = sb.toString();
             for (SearchHit hit : searchResponse.getHits().getHits()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 31c8e5bc3d457..e1ba661eb24d4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -33,6 +33,8 @@
 import org.apache.logging.log4j.status.StatusConsoleListener;
 import org.apache.logging.log4j.status.StatusData;
 import org.apache.logging.log4j.status.StatusLogger;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.tests.util.TestRuleMarkFailure;
@@ -208,6 +210,7 @@
 import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 import static org.hamcrest.Matchers.anyOf;
@@ -2637,4 +2640,43 @@ public static void ensureAllContextsReleased(SearchService searchService) {
             throw new AssertionError("Failed to verify search contexts", e);
         }
     }
+
+    /**
+     * Create a new searcher over the reader. This searcher might randomly use threads.
+     * Provides the same functionality as {@link LuceneTestCase#newSearcher(IndexReader)},
+     * with the only difference that concurrency will only ever be inter-segment and never intra-segment.
+     */
+    public static IndexSearcher newSearcher(IndexReader r) {
+        return newSearcher(r, true);
+    }
+
+    /**
+     * Create a new searcher over the reader. This searcher might randomly use threads.
+     * Provides the same functionality as {@link LuceneTestCase#newSearcher(IndexReader, boolean)},
+     * with the only difference that concurrency will only ever be inter-segment and never intra-segment.
+     */
+    public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) {
+        return newSearcher(r, maybeWrap, true);
+    }
+
+    /**
+     * Create a new searcher over the reader. This searcher might randomly use threads.
+     * Provides the same functionality as {@link LuceneTestCase#newSearcher(IndexReader, boolean, boolean)},
+     * with the only difference that concurrency will only ever be inter-segment and never intra-segment.
+     */
+    public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap, boolean wrapWithAssertions) {
+        return newSearcher(r, maybeWrap, wrapWithAssertions, randomBoolean());
+    }
+
+    /**
+     * Create a new searcher over the reader.
+     * Provides the same functionality as {@link LuceneTestCase#newSearcher(IndexReader, boolean, boolean, boolean)},
+     * with the only difference that concurrency will only ever be inter-segment and never intra-segment.
+     */
+    public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap, boolean wrapWithAssertions, boolean useThreads) {
+        if (useThreads) {
+            return newSearcher(r, maybeWrap, wrapWithAssertions, Concurrency.INTER_SEGMENT);
+        }
+        return newSearcher(r, maybeWrap, wrapWithAssertions, Concurrency.NONE);
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
deleted file mode 100644
index 9e2f8c931c84a..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-package org.elasticsearch.test.disruption;
-
-import org.elasticsearch.core.TimeValue;
-
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Simulates irregular long gc intervals.
- */
-public class IntermittentLongGCDisruption extends LongGCDisruption {
-
-    volatile boolean disrupting;
-    volatile Thread worker;
-
-    final long intervalBetweenDelaysMin;
-    final long intervalBetweenDelaysMax;
-    final long delayDurationMin;
-    final long delayDurationMax;
-
-    public IntermittentLongGCDisruption(
-        Random random,
-        String disruptedNode,
-        long intervalBetweenDelaysMin,
-        long intervalBetweenDelaysMax,
-        long delayDurationMin,
-        long delayDurationMax
-    ) {
-        super(random, disruptedNode);
-        this.intervalBetweenDelaysMin = intervalBetweenDelaysMin;
-        this.intervalBetweenDelaysMax = intervalBetweenDelaysMax;
-        this.delayDurationMin = delayDurationMin;
-        this.delayDurationMax = delayDurationMax;
-    }
-
-    static final AtomicInteger thread_ids = new AtomicInteger();
-
-    @Override
-    public void startDisrupting() {
-        disrupting = true;
-        worker = new Thread(new BackgroundWorker(), "long_gc_simulation_" + thread_ids.incrementAndGet());
-        worker.setDaemon(true);
-        worker.start();
-    }
-
-    @Override
-    public void stopDisrupting() {
-        if (worker == null) {
-            return;
-        }
-        logger.info("stopping long GCs on [{}]", disruptedNode);
-        disrupting = false;
-        worker.interrupt();
-        try {
-            worker.join(2 * (intervalBetweenDelaysMax + delayDurationMax));
-        } catch (InterruptedException e) {
-            logger.info("background thread failed to stop");
-        }
-        worker = null;
-    }
-
-    private void simulateLongGC(final TimeValue duration) throws InterruptedException {
-        logger.info("node [{}] goes into GC for for [{}]", disruptedNode, duration);
-        final Set<Thread> nodeThreads = new HashSet<>();
-        try {
-            while (suspendThreads(nodeThreads))
-                ;
-            if (nodeThreads.isEmpty() == false) {
-                Thread.sleep(duration.millis());
-            }
-        } finally {
-            logger.info("node [{}] resumes from GC", disruptedNode);
-            resumeThreads(nodeThreads);
-        }
-    }
-
-    class BackgroundWorker implements Runnable {
-
-        @Override
-        public void run() {
-            while (disrupting) {
-                try {
-                    TimeValue duration = new TimeValue(delayDurationMin + random.nextInt((int) (delayDurationMax - delayDurationMin)));
-                    simulateLongGC(duration);
-
-                    duration = new TimeValue(
-                        intervalBetweenDelaysMin + random.nextInt((int) (intervalBetweenDelaysMax - intervalBetweenDelaysMin))
-                    );
-                    if (disrupting) {
-                        Thread.sleep(duration.millis());
-                    }
-                } catch (InterruptedException e) {} catch (Exception e) {
-                    logger.error("error in background worker", e);
-                }
-            }
-        }
-    }
-
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
deleted file mode 100644
index dce9e2600d0a6..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.test.disruption;
-
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.test.InternalTestCluster;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadInfo;
-import java.lang.management.ThreadMXBean;
-import java.util.Arrays;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-/**
- * Suspends all threads on the specified node in order to simulate a long gc.
- */
-public class LongGCDisruption extends SingleNodeDisruption {
-
-    private static final Pattern[] unsafeClasses = new Pattern[] {
-        // logging has shared JVM locks; we may suspend a thread and block other nodes from doing their thing
-        Pattern.compile("logging\\.log4j"),
-        // security manager is shared across all nodes and it uses synchronized maps internally
-        Pattern.compile("java\\.lang\\.SecurityManager"),
-        // SecureRandom instance from SecureRandomHolder class is shared by all nodes
-        Pattern.compile("java\\.security\\.SecureRandom"),
-        // Lucene's WindowsFS is shared across nodes and contains some coarse synchronization
-        Pattern.compile("org\\.apache\\.lucene\\.tests\\.mockfile\\.WindowsFS") };
-
-    private static final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
-
-    protected final String disruptedNode;
-    private Set<Thread> suspendedThreads;
-    private Thread blockDetectionThread;
-
-    private final AtomicBoolean sawSlowSuspendBug = new AtomicBoolean(false);
-
-    public LongGCDisruption(Random random, String disruptedNode) {
-        super(random);
-        this.disruptedNode = disruptedNode;
-    }
-
-    /**
-     * Checks if during disruption we ran into a known JVM issue that makes {@link Thread#suspend()} calls block for multiple seconds
-     * was observed.
-     * @see <a href=https://bugs.openjdk.java.net/browse/JDK-8218446>JDK-8218446</a>
-     * @return true if during thread suspending a call to {@link Thread#suspend()} took more than 3s
-     */
-    public boolean sawSlowSuspendBug() {
-        return sawSlowSuspendBug.get();
-    }
-
-    @Override
-    public synchronized void startDisrupting() {
-        if (suspendedThreads == null) {
-            boolean success = false;
-            try {
-                suspendedThreads = ConcurrentHashMap.newKeySet();
-
-                final String currentThreadName = Thread.currentThread().getName();
-                assert isDisruptedNodeThread(currentThreadName) == false
-                    : "current thread match pattern. thread name: " + currentThreadName + ", node: " + disruptedNode;
-                // we spawn a background thread to protect against deadlock which can happen
-                // if there are shared resources between caller thread and suspended threads
-                // see unsafeClasses to how to avoid that
-                final AtomicReference<Exception> suspendingError = new AtomicReference<>();
-                final Thread suspendingThread = new Thread(new AbstractRunnable() {
-                    @Override
-                    public void onFailure(Exception e) {
-                        suspendingError.set(e);
-                    }
-
-                    @Override
-                    protected void doRun() throws Exception {
-                        // keep trying to suspend threads, until no new threads are discovered.
-                        while (suspendThreads(suspendedThreads)) {
-                            if (Thread.interrupted()) {
-                                return;
-                            }
-                        }
-                    }
-                });
-                suspendingThread.setName(currentThreadName + "[LongGCDisruption][threadSuspender]");
-                suspendingThread.start();
-                try {
-                    suspendingThread.join(getSuspendingTimeoutInMillis());
-                } catch (InterruptedException e) {
-                    suspendingThread.interrupt(); // best effort to signal suspending
-                    throw new RuntimeException(e);
-                }
-                if (suspendingError.get() != null) {
-                    throw new RuntimeException("unknown error while suspending threads", suspendingError.get());
-                }
-                if (suspendingThread.isAlive()) {
-                    logger.warn(
-                        """
-                            failed to suspend node [{}]'s threads within [{}] millis. Suspending thread stack trace:
-                             {}
-                            Threads that weren't suspended:
-                             {}""",
-                        disruptedNode,
-                        getSuspendingTimeoutInMillis(),
-                        stackTrace(suspendingThread.getStackTrace()),
-                        suspendedThreads.stream()
-                            .map(t -> t.getName() + "\n----\n" + stackTrace(t.getStackTrace()))
-                            .collect(Collectors.joining("\n"))
-                    );
-                    suspendingThread.interrupt(); // best effort;
-                    try {
-                        /*
-                         * We need to join on the suspending thread in case it has suspended a thread that is in a critical section and
-                         * needs to be resumed.
-                         */
-                        suspendingThread.join();
-                    } catch (InterruptedException e) {
-                        throw new RuntimeException(e);
-                    }
-                    throw new RuntimeException("suspending node threads took too long");
-                }
-                // block detection checks if other threads are blocked waiting on an object that is held by one
-                // of the threads that was suspended
-                if (isBlockDetectionSupported()) {
-                    blockDetectionThread = new Thread(new AbstractRunnable() {
-                        @Override
-                        public void onFailure(Exception e) {
-                            if (e instanceof InterruptedException == false) {
-                                throw new AssertionError("unexpected exception in blockDetectionThread", e);
-                            }
-                        }
-
-                        @Override
-                        protected void doRun() throws Exception {
-                            while (Thread.currentThread().isInterrupted() == false) {
-                                ThreadInfo[] threadInfos = threadBean.dumpAllThreads(true, true);
-                                for (ThreadInfo threadInfo : threadInfos) {
-                                    if (isDisruptedNodeThread(threadInfo.getThreadName()) == false
-                                        && threadInfo.getLockOwnerName() != null
-                                        && isDisruptedNodeThread(threadInfo.getLockOwnerName())) {
-
-                                        // find ThreadInfo object of the blocking thread (if available)
-                                        ThreadInfo blockingThreadInfo = null;
-                                        for (ThreadInfo otherThreadInfo : threadInfos) {
-                                            if (otherThreadInfo.getThreadId() == threadInfo.getLockOwnerId()) {
-                                                blockingThreadInfo = otherThreadInfo;
-                                                break;
-                                            }
-                                        }
-                                        onBlockDetected(threadInfo, blockingThreadInfo);
-                                    }
-                                }
-                                Thread.sleep(getBlockDetectionIntervalInMillis());
-                            }
-                        }
-                    });
-                    blockDetectionThread.setName(currentThreadName + "[LongGCDisruption][blockDetection]");
-                    blockDetectionThread.start();
-                }
-                success = true;
-            } finally {
-                if (success == false) {
-                    stopBlockDetection();
-                    // resume threads if failed
-                    resumeThreads(suspendedThreads);
-                    suspendedThreads = null;
-                }
-            }
-        } else {
-            throw new IllegalStateException("can't disrupt twice, call stopDisrupting() first");
-        }
-    }
-
-    public boolean isDisruptedNodeThread(String threadName) {
-        return threadName.contains("[" + disruptedNode + "]");
-    }
-
-    private static String stackTrace(StackTraceElement[] stackTraceElements) {
-        return Arrays.stream(stackTraceElements).map(Object::toString).collect(Collectors.joining("\n"));
-    }
-
-    @Override
-    public synchronized void stopDisrupting() {
-        stopBlockDetection();
-        if (suspendedThreads != null) {
-            resumeThreads(suspendedThreads);
-            suspendedThreads = null;
-        }
-    }
-
-    private void stopBlockDetection() {
-        if (blockDetectionThread != null) {
-            try {
-                blockDetectionThread.interrupt(); // best effort
-                blockDetectionThread.join(getSuspendingTimeoutInMillis());
-            } catch (InterruptedException e) {
-                throw new RuntimeException(e);
-            }
-            blockDetectionThread = null;
-        }
-    }
-
-    @Override
-    public void removeAndEnsureHealthy(InternalTestCluster cluster) {
-        removeFromCluster(cluster);
-        ensureNodeCount(cluster);
-    }
-
-    @Override
-    public TimeValue expectedTimeToHeal() {
-        return TimeValue.timeValueMillis(0);
-    }
-
-    /**
-     * resolves all threads belonging to given node and suspends them if their current stack trace
-     * is "safe". Threads are added to nodeThreads if suspended.
-     *
-     * returns true if some live threads were found. The caller is expected to call this method
-     * until no more "live" are found.
-     */
-    @SuppressWarnings("deprecation") // suspends/resumes threads intentionally
-    @SuppressForbidden(reason = "suspends/resumes threads intentionally")
-    protected boolean suspendThreads(Set<Thread> nodeThreads) {
-        Thread[] allThreads = null;
-        while (allThreads == null) {
-            allThreads = new Thread[Thread.activeCount()];
-            if (Thread.enumerate(allThreads) > allThreads.length) {
-                // we didn't make enough space, retry
-                allThreads = null;
-            }
-        }
-        boolean liveThreadsFound = false;
-        for (Thread thread : allThreads) {
-            if (thread == null) {
-                continue;
-            }
-            String threadName = thread.getName();
-            if (isDisruptedNodeThread(threadName)) {
-                if (thread.isAlive() && nodeThreads.add(thread)) {
-                    liveThreadsFound = true;
-                    logger.trace("suspending thread [{}]", threadName);
-                    // we assume it is not safe to suspend the thread
-                    boolean safe = false;
-                    try {
-                        /*
-                         * At the bottom of this try-block we will know whether or not it is safe to suspend the thread; we start by
-                         * assuming that it is safe.
-                         */
-                        boolean definitelySafe = true;
-                        final long startTime = System.nanoTime();
-                        thread.suspend();
-                        if (System.nanoTime() - startTime > TimeUnit.SECONDS.toNanos(3L)) {
-                            sawSlowSuspendBug.set(true);
-                        }
-                        // double check the thread is not in a shared resource like logging; if so, let it go and come back
-                        safe: for (StackTraceElement stackElement : thread.getStackTrace()) {
-                            String className = stackElement.getClassName();
-                            for (Pattern unsafePattern : getUnsafeClasses()) {
-                                if (unsafePattern.matcher(className).find()) {
-                                    // it is definitely not safe to suspend the thread
-                                    definitelySafe = false;
-                                    break safe;
-                                }
-                            }
-                        }
-                        safe = definitelySafe;
-                    } finally {
-                        if (safe == false) {
-                            /*
-                             * Do not log before resuming as we might be interrupted while logging in which case we will throw an
-                             * interrupted exception and never resume the suspended thread that is in a critical section. Also, logging
-                             * before resuming makes for confusing log messages if we never hit the resume.
-                             */
-                            thread.resume();
-                            logger.trace("resumed thread [{}] as it is in a critical section", threadName);
-                            nodeThreads.remove(thread);
-                        }
-                    }
-                }
-            }
-        }
-        return liveThreadsFound;
-    }
-
-    // for testing
-    protected Pattern[] getUnsafeClasses() {
-        return unsafeClasses;
-    }
-
-    // for testing
-    protected long getSuspendingTimeoutInMillis() {
-        return TimeValue.timeValueSeconds(30).getMillis();
-    }
-
-    public boolean isBlockDetectionSupported() {
-        return threadBean.isObjectMonitorUsageSupported() && threadBean.isSynchronizerUsageSupported();
-    }
-
-    // for testing
-    protected long getBlockDetectionIntervalInMillis() {
-        return 3000L;
-    }
-
-    // for testing
-    protected void onBlockDetected(ThreadInfo blockedThread, @Nullable ThreadInfo blockingThread) {
-        String blockedThreadStackTrace = stackTrace(blockedThread.getStackTrace());
-        String blockingThreadStackTrace = blockingThread != null ? stackTrace(blockingThread.getStackTrace()) : "not available";
-        throw new AssertionError(
-            "Thread ["
-                + blockedThread.getThreadName()
-                + "] is blocked waiting on the resource ["
-                + blockedThread.getLockInfo()
-                + "] held by the suspended thread ["
-                + blockedThread.getLockOwnerName()
-                + "] of the disrupted node ["
-                + disruptedNode
-                + "].\n"
-                + "Please add this occurrence to the unsafeClasses list in ["
-                + LongGCDisruption.class.getName()
-                + "].\n"
-                + "Stack trace of blocked thread: "
-                + blockedThreadStackTrace
-                + "\n"
-                + "Stack trace of blocking thread: "
-                + blockingThreadStackTrace
-        );
-    }
-
-    @SuppressWarnings("deprecation") // suspends/resumes threads intentionally
-    @SuppressForbidden(reason = "suspends/resumes threads intentionally")
-    protected void resumeThreads(Set<Thread> threads) {
-        for (Thread thread : threads) {
-            thread.resume();
-        }
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
index db30f6e91f039..42439b5d5785d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
@@ -86,13 +86,6 @@ public Terms terms(String field) throws IOException {
         return terms;
     }
 
-    @Override
-    public Fields getTermVectors(int docID) throws IOException {
-        Fields fields = super.getTermVectors(docID);
-        thrower.maybeThrow(Flags.TermVectors);
-        return fields == null ? null : new ThrowingFields(fields, thrower);
-    }
-
     /**
      * Wraps a Fields but with additional asserts
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index ad61a63f7c46e..5851fc709d14a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -14,7 +14,9 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.RequestBuilder;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
@@ -312,7 +314,7 @@ public static void assertHitCount(ActionFuture<SearchResponse> responseFuture, l
 
     public static void assertHitCount(SearchResponse countResponse, long expectedHitCount) {
         final TotalHits totalHits = countResponse.getHits().getTotalHits();
-        if (totalHits.relation != TotalHits.Relation.EQUAL_TO || totalHits.value != expectedHitCount) {
+        if (totalHits.relation() != TotalHits.Relation.EQUAL_TO || totalHits.value() != expectedHitCount) {
             fail("Count is " + totalHits + " but " + expectedHitCount + " was expected. " + formatShardStatus(countResponse));
         }
     }
@@ -344,7 +346,7 @@ public static void assertFourthHit(SearchResponse searchResponse, Matcher<Search
 
     public static void assertSearchHit(SearchResponse searchResponse, int number, Matcher<SearchHit> matcher) {
         assertThat("SearchHit number must be greater than 0", number, greaterThan(0));
-        assertThat(searchResponse.getHits().getTotalHits().value, greaterThanOrEqualTo((long) number));
+        assertThat(searchResponse.getHits().getTotalHits().value(), greaterThanOrEqualTo((long) number));
         assertThat(searchResponse.getHits().getAt(number - 1), matcher);
     }
 
@@ -407,13 +409,13 @@ public static void assertScrollResponsesAndHitCount(
         responses.add(scrollResponse);
         int retrievedDocsCount = 0;
         try {
-            assertThat(scrollResponse.getHits().getTotalHits().value, equalTo((long) expectedTotalHitCount));
+            assertThat(scrollResponse.getHits().getTotalHits().value(), equalTo((long) expectedTotalHitCount));
             retrievedDocsCount += scrollResponse.getHits().getHits().length;
             responseConsumer.accept(responses.size(), scrollResponse);
             while (scrollResponse.getHits().getHits().length > 0) {
                 scrollResponse = client.prepareSearchScroll(scrollResponse.getScrollId()).setScroll(keepAlive).get();
                 responses.add(scrollResponse);
-                assertThat(scrollResponse.getHits().getTotalHits().value, equalTo((long) expectedTotalHitCount));
+                assertThat(scrollResponse.getHits().getTotalHits().value(), equalTo((long) expectedTotalHitCount));
                 retrievedDocsCount += scrollResponse.getHits().getHits().length;
                 responseConsumer.accept(responses.size(), scrollResponse);
             }
@@ -702,8 +704,8 @@ public static <T extends Query> T assertBooleanSubQuery(Query query, Class<T> su
         assertThat(query, instanceOf(BooleanQuery.class));
         BooleanQuery q = (BooleanQuery) query;
         assertThat(q.clauses(), hasSize(greaterThan(i)));
-        assertThat(q.clauses().get(i).getQuery(), instanceOf(subqueryType));
-        return subqueryType.cast(q.clauses().get(i).getQuery());
+        assertThat(q.clauses().get(i).query(), instanceOf(subqueryType));
+        return subqueryType.cast(q.clauses().get(i).query());
     }
 
     /**
@@ -873,6 +875,73 @@ public static void awaitLatch(CountDownLatch latch, long timeout, TimeUnit unit)
         assertThat(message, isCountedDown, is(true));
     }
 
+    /**
+     * Check the response of a client request to ensure it has a warning header that contains the provided string
+     *
+     * Currently, this allows a fixed 10 seconds for response to be received
+     * @param client Client making the request - Required to access the response headers
+     * @param requestBuilder Request to be made
+     * @param toMatch String to match in the warning headers
+     * @param <T> Type of the response
+     * @throws InterruptedException If the request times out
+     */
+    public static <T extends ActionResponse> void assertWarningHeaderOnResponse(
+        Client client,
+        ActionRequestBuilder<?, T> requestBuilder,
+        String toMatch
+    ) throws InterruptedException {
+        assertWarningHeaderMatchOnResponse(client, requestBuilder, hasItem(containsString(toMatch)));
+    }
+
+    /**
+     * Check the response of a client request to ensure it does not have a warning header that contains the provided string
+     *
+     * Currently, this allows a fixed 10 seconds for response to be received
+     * @param client Client making the request - Required to access the response headers
+     * @param requestBuilder Request to be made
+     * @param toMatch String to not match in the warning headers
+     * @param <T> Type of the response
+     * @throws InterruptedException If the request times out
+     */
+    public static <T extends ActionResponse> void assertNoWarningHeaderOnResponse(
+        Client client,
+        ActionRequestBuilder<?, T> requestBuilder,
+        String toMatch
+    ) throws InterruptedException {
+        assertWarningHeaderMatchOnResponse(client, requestBuilder, not(hasItem(containsString(toMatch))));
+    }
+
+    private static <T extends ActionResponse> void assertWarningHeaderMatchOnResponse(
+        Client client,
+        ActionRequestBuilder<?, T> requestBuilder,
+        Matcher<? super List<String>> matcher
+    ) throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(1);
+        requestBuilder.execute(new ActionListener<>() {
+            @Override
+            public void onResponse(T response) {
+                try {
+                    final var warningHeaders = client.threadPool().getThreadContext().getResponseHeaders().get("Warning");
+                    assertThat(warningHeaders, matcher);
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    throw new AssertionError("Failed to execute request", e);
+                } finally {
+                    latch.countDown();
+                }
+            }
+        });
+        if (latch.await(10, TimeUnit.SECONDS) == false) {
+            fail("Did not receive request response before timeout");
+        }
+    }
+
     /**
      * Compares two maps recursively, using arrays comparisons for byte[] through Arrays.equals(byte[], byte[])
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index d17016f850300..22f93e6bda61f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -1896,7 +1896,11 @@ protected Map<String, Object> getIndexMappingAsMap(String index) throws IOExcept
     }
 
     protected static boolean indexExists(String index) throws IOException {
-        Response response = client().performRequest(new Request("HEAD", "/" + index));
+        return indexExists(client(), index);
+    }
+
+    protected static boolean indexExists(RestClient client, String index) throws IOException {
+        Response response = client.performRequest(new Request("HEAD", "/" + index));
         return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode();
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 57a6d1e09c52d..18c591166e720 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
@@ -50,7 +49,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BytesTransportRequest;
 import org.elasticsearch.transport.ClusterConnectionManager;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.ConnectionProfile;
@@ -80,6 +78,8 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
@@ -104,6 +104,7 @@ public class MockTransportService extends TransportService {
     private final Map<DiscoveryNode, List<Transport.Connection>> openConnections = new HashMap<>();
 
     private final List<Runnable> onStopListeners = new CopyOnWriteArrayList<>();
+    private final AtomicReference<Consumer<Transport.Connection>> onConnectionClosedCallback = new AtomicReference<>();
 
     public static class TestPlugin extends Plugin {
         @Override
@@ -583,13 +584,8 @@ public void sendRequest(
                 // poor mans request cloning...
                 BytesStreamOutput bStream = new BytesStreamOutput();
                 request.writeTo(bStream);
-                final TransportRequest clonedRequest;
-                if (request instanceof BytesTransportRequest) {
-                    clonedRequest = copyRawBytesForBwC(bStream);
-                } else {
-                    RequestHandlerRegistry<?> reg = MockTransportService.this.getRequestHandler(action);
-                    clonedRequest = reg.newRequest(bStream.bytes().streamInput());
-                }
+                RequestHandlerRegistry<?> reg = MockTransportService.this.getRequestHandler(action);
+                final TransportRequest clonedRequest = reg.newRequest(bStream.bytes().streamInput());
                 assert clonedRequest.getClass().equals(MasterNodeRequestHelper.unwrapTermOverride(request).getClass())
                     : clonedRequest + " vs " + request;
 
@@ -637,15 +633,6 @@ protected void doRun() throws IOException {
                 }
             }
 
-            // Some request handlers read back a BytesTransportRequest
-            // into a different class that cannot be re-serialized (i.e. JOIN_VALIDATE_ACTION_NAME),
-            // in those cases we just copy the raw bytes back to a BytesTransportRequest.
-            // This is only needed for the BwC for JOIN_VALIDATE_ACTION_NAME and can be removed in the next major
-            @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-            private static TransportRequest copyRawBytesForBwC(BytesStreamOutput bStream) throws IOException {
-                return new BytesTransportRequest(bStream.bytes().streamInput());
-            }
-
             @Override
             public void clearCallback() {
                 synchronized (this) {
@@ -788,6 +775,19 @@ public void openConnection(DiscoveryNode node, ConnectionProfile connectionProfi
         }));
     }
 
+    public void setOnConnectionClosedCallback(Consumer<Transport.Connection> callback) {
+        onConnectionClosedCallback.set(callback);
+    }
+
+    @Override
+    public void onConnectionClosed(Transport.Connection connection) {
+        final Consumer<Transport.Connection> callback = onConnectionClosedCallback.get();
+        if (callback != null) {
+            callback.accept(connection);
+        }
+        super.onConnectionClosed(connection);
+    }
+
     public void addOnStopListener(Runnable listener) {
         onStopListeners.add(listener);
     }
@@ -814,9 +814,4 @@ protected void doClose() throws IOException {
             assertTrue(ThreadPool.terminate(testExecutor, 10, TimeUnit.SECONDS));
         }
     }
-
-    public DiscoveryNode getLocalDiscoNode() {
-        return this.getLocalNode();
-    }
-
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 34f67ac78a41c..4595fbf286077 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -668,7 +668,7 @@ public void testVoidMessageCompressed() throws Exception {
                 )
                 .build();
             ConnectionProfile connectionProfile = ConnectionProfile.buildDefaultConnectionProfile(settingsWithCompress);
-            connectToNode(serviceC, serviceA.getLocalDiscoNode(), connectionProfile);
+            connectToNode(serviceC, serviceA.getLocalNode(), connectionProfile);
 
             Future<TransportResponse.Empty> res = submitRequest(
                 serviceC,
@@ -725,7 +725,7 @@ public void testHelloWorldCompressed() throws Exception {
                 )
                 .build();
             ConnectionProfile connectionProfile = ConnectionProfile.buildDefaultConnectionProfile(settingsWithCompress);
-            connectToNode(serviceC, serviceA.getLocalDiscoNode(), connectionProfile);
+            connectToNode(serviceC, serviceA.getLocalNode(), connectionProfile);
 
             Future<StringMessageResponse> res = submitRequest(
                 serviceC,
@@ -795,8 +795,8 @@ public void testIndexingDataCompression() throws Exception {
                 )
                 .build();
             ConnectionProfile connectionProfile = ConnectionProfile.buildDefaultConnectionProfile(settingsWithCompress);
-            connectToNode(serviceC, serviceA.getLocalDiscoNode(), connectionProfile);
-            connectToNode(serviceA, serviceC.getLocalDiscoNode(), connectionProfile);
+            connectToNode(serviceC, serviceA.getLocalNode(), connectionProfile);
+            connectToNode(serviceA, serviceC.getLocalNode(), connectionProfile);
 
             TransportResponseHandler<StringMessageResponse> responseHandler = new TransportResponseHandler<>() {
                 @Override
@@ -821,14 +821,14 @@ public void handleException(TransportException exp) {
 
             Future<StringMessageResponse> compressed = submitRequest(
                 serviceC,
-                serviceA.getLocalDiscoNode(),
+                serviceA.getLocalNode(),
                 "internal:sayHello",
                 new StringMessageRequest(text, -1, true),
                 responseHandler
             );
             Future<StringMessageResponse> uncompressed = submitRequest(
                 serviceA,
-                serviceC.getLocalDiscoNode(),
+                serviceC.getLocalNode(),
                 "internal:sayHello",
                 new StringMessageRequest(text, -1, false),
                 responseHandler
@@ -1049,7 +1049,7 @@ public void onAfter() {
                     ignoringRequestHandler
                 );
                 serviceB = newService;
-                nodeB = newService.getLocalDiscoNode();
+                nodeB = newService.getLocalNode();
                 connectToNode(serviceB, nodeA);
                 connectToNode(serviceA, nodeB);
             } else if (serviceA.nodeConnected(nodeB)) {
@@ -3419,7 +3419,7 @@ public <T extends TransportResponse> void sendRequest(
         ) {
             final CountDownLatch latch = new CountDownLatch(1);
             serviceC.connectToNode(
-                serviceA.getLocalDiscoNode(),
+                serviceA.getLocalNode(),
                 ConnectionProfile.buildDefaultConnectionProfile(Settings.EMPTY),
                 new ActionListener<>() {
                     @Override
diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java
similarity index 65%
rename from test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java
rename to test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java
index 1178a32e46d31..bac61daf554d5 100644
--- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java
@@ -17,25 +17,27 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
-public class DataGeneratorSnapshotTests extends ESTestCase {
+public class DataGenerationSnapshotTests extends ESTestCase {
     public void testSnapshot() throws Exception {
-        var dataGenerator = new DataGenerator(
-            DataGeneratorSpecification.builder()
-                .withDataSourceHandlers(List.of(new DataSourceOverrides()))
-                .withMaxFieldCountPerLevel(5)
-                .withMaxObjectDepth(2)
-                .build()
-        );
+        var specification = DataGeneratorSpecification.builder()
+            .withDataSourceHandlers(List.of(new DataSourceOverrides()))
+            .withMaxFieldCountPerLevel(5)
+            .withMaxObjectDepth(2)
+            .build();
 
-        var mapping = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint();
-        dataGenerator.writeMapping(mapping);
+        var template = new TemplateGenerator(specification).generate();
+        var mapping = new MappingGenerator(specification).generate(template);
 
-        var document = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint();
-        dataGenerator.generateDocument(document);
+        var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint();
+        mappingXContent.map(mapping.raw());
+
+        var documentXContent = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint();
+        documentXContent.map(new DocumentGenerator(specification).generate(template, mapping));
 
         var expectedMapping = """
             {
@@ -49,53 +51,66 @@ public void testSnapshot() throws Exception {
                         "dynamic" : "false",
                         "properties" : {
                           "f3" : {
-                            "type" : "keyword",
-                            "store" : "true"
+                            "store" : "true",
+                            "type" : "keyword"
                           },
                           "f4" : {
-                            "type" : "long",
-                            "index" : "false"
+                            "index" : "false",
+                            "type" : "long"
                           }
-                        }
+                        },
+                        "type" : "object"
                       },
                       "f5" : {
                         "dynamic" : "false",
                         "properties" : {
                           "f6" : {
-                            "type" : "keyword",
-                            "store" : "true"
+                            "store" : "true",
+                            "type" : "keyword"
                           },
                           "f7" : {
-                            "type" : "long",
-                            "index" : "false"
+                            "dynamic" : "false",
+                            "properties" : {
+                              "f8" : {
+                                "index" : "false",
+                                "type" : "long"
+                              },
+                              "f9" : {
+                                "store" : "true",
+                                "type" : "keyword"
+                              }
+                            },
+                            "type" : "nested"
                           }
-                        }
+                        },
+                        "type" : "object"
                       }
-                    }
+                    },
+                    "type" : "object"
                   },
-                  "f8" : {
-                    "type" : "nested",
+                  "f10" : {
                     "dynamic" : "false",
                     "properties" : {
-                      "f9" : {
-                        "type" : "nested",
+                      "f11" : {
                         "dynamic" : "false",
                         "properties" : {
-                          "f10" : {
-                            "type" : "keyword",
-                            "store" : "true"
+                          "f12" : {
+                            "index" : "false",
+                            "type" : "long"
                           },
-                          "f11" : {
-                            "type" : "long",
-                            "index" : "false"
+                          "f13" : {
+                            "store" : "true",
+                            "type" : "keyword"
                           }
-                        }
+                        },
+                        "type" : "nested"
                       },
-                      "f12" : {
-                        "type" : "keyword",
-                        "store" : "true"
+                      "f14" : {
+                        "index" : "false",
+                        "type" : "long"
                       }
-                    }
+                    },
+                    "type" : "nested"
                   }
                 }
               }
@@ -103,48 +118,48 @@ public void testSnapshot() throws Exception {
 
         var expectedDocument = """
             {
-              "f1" : [
+              "f1" : {
+                "f2" : {
+                  "f3" : null,
+                  "f4" : 3
+                },
+                "f5" : {
+                  "f6" : [
+                    "string4",
+                    "string5"
+                  ],
+                  "f7" : {
+                    "f8" : null,
+                    "f9" : "string6"
+                  }
+                }
+              },
+              "f10" : [
                 {
-                  "f2" : {
-                    "f3" : [
+                  "f11" : {
+                    "f12" : [
                       null,
-                      "string1"
+                      0
                     ],
-                    "f4" : 0
+                    "f13" : "string1"
                   },
-                  "f5" : {
-                    "f6" : "string2",
-                    "f7" : null
-                  }
+                  "f14" : 1
                 },
                 {
-                  "f2" : {
-                    "f3" : [
-                      "string3",
-                      "string4"
-                    ],
-                    "f4" : 1
+                  "f11" : {
+                    "f12" : null,
+                    "f13" : [
+                      "string2",
+                      "string3"
+                    ]
                   },
-                  "f5" : {
-                    "f6" : null,
-                    "f7" : 2
-                  }
+                  "f14" : 2
                 }
-              ],
-              "f8" : {
-                "f9" : {
-                  "f10" : [
-                    "string5",
-                    "string6"
-                  ],
-                  "f11" : null
-                },
-                "f12" : "string7"
-              }
+              ]
             }""";
 
-        assertEquals(expectedMapping, Strings.toString(mapping));
-        assertEquals(expectedDocument, Strings.toString(document));
+        assertEquals(expectedMapping, Strings.toString(mappingXContent));
+        assertEquals(expectedDocument, Strings.toString(documentXContent));
     }
 
     private static class DataSourceOverrides implements DataSourceHandler {
@@ -173,7 +188,6 @@ public DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper reque
 
         @Override
         public DataSourceResponse.ArrayWrapper handle(DataSourceRequest.ArrayWrapper request) {
-
             return new DataSourceResponse.ArrayWrapper((values) -> () -> {
                 if (generateArrayChecks++ % 4 == 0) {
                     // we have nulls so can't use List.of
@@ -206,14 +220,19 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG
             return new DataSourceResponse.FieldTypeGenerator(() -> {
                 if (fieldType == FieldType.KEYWORD) {
                     fieldType = FieldType.LONG;
-                    return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD, false);
+                    return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD);
                 }
 
                 fieldType = FieldType.KEYWORD;
-                return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG, false);
+                return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG);
             });
         }
 
+        @Override
+        public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) {
+            return new DataSourceResponse.DynamicMappingGenerator((ignored) -> false);
+        }
+
         @Override
         public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
             if (request.fieldType() == FieldType.KEYWORD) {
@@ -229,7 +248,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
 
         @Override
         public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
-            return new DataSourceResponse.ObjectMappingParametersGenerator(() -> Map.of("dynamic", "false"));
+            return new DataSourceResponse.ObjectMappingParametersGenerator(() -> new HashMap<>(Map.of("dynamic", "false")));
         }
     }
 
diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java
similarity index 69%
rename from test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java
rename to test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java
index b5fa66bf3006b..d9750328ff3fa 100644
--- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
-import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -28,20 +27,8 @@
 import java.util.List;
 import java.util.Optional;
 
-public class DataGeneratorTests extends ESTestCase {
-    public void testDataGeneratorSanity() throws IOException {
-        var dataGenerator = new DataGenerator(DataGeneratorSpecification.buildDefault());
-
-        var mapping = XContentBuilder.builder(XContentType.JSON.xContent());
-        dataGenerator.writeMapping(mapping);
-
-        for (int i = 0; i < 1000; i++) {
-            var document = XContentBuilder.builder(XContentType.JSON.xContent());
-            dataGenerator.generateDocument(document);
-        }
-    }
-
-    public void testDataGeneratorProducesValidMappingAndDocument() throws IOException {
+public class DataGenerationTests extends ESTestCase {
+    public void testDataGenerationProducesValidMappingAndDocument() throws IOException {
         // Make sure objects, nested objects and all field types are covered.
         var testChildFieldGenerator = new DataSourceResponse.ChildFieldGenerator() {
             private boolean dynamicSubObjectCovered = false;
@@ -101,43 +88,34 @@ public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFiel
 
             @Override
             public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
-                if (request.dynamicMapping() == DynamicMapping.FORBIDDEN || request.dynamicMapping() == DynamicMapping.SUPPORTED) {
-                    return new DataSourceResponse.FieldTypeGenerator(
-                        () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(
-                            FieldType.values()[generatedFields++ % FieldType.values().length],
-                            false
-                        )
-                    );
-                }
-
-                return new DataSourceResponse.FieldTypeGenerator(() -> {
-                    var fieldType = FieldType.values()[generatedFields++ % FieldType.values().length];
-                    // Does not really work with dynamic mapping.
-                    if (fieldType == FieldType.UNSIGNED_LONG) {
-                        fieldType = FieldType.values()[generatedFields++ % FieldType.values().length];
-                    }
+                return new DataSourceResponse.FieldTypeGenerator(
+                    () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(
+                        FieldType.values()[generatedFields++ % FieldType.values().length]
+                    )
+                );
 
-                    return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(fieldType, true);
-                });
             }
         };
 
-        var dataGenerator = new DataGenerator(
-            DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).build()
-        );
+        var specification = DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).build();
+
+        var documentGenerator = new DocumentGenerator(specification);
 
-        var mapping = XContentBuilder.builder(XContentType.JSON.xContent());
-        dataGenerator.writeMapping(mapping);
+        var template = new TemplateGenerator(specification).generate();
+        var mapping = new MappingGenerator(specification).generate(template);
+
+        var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent());
+        mappingXContent.map(mapping.raw());
 
         var mappingService = new MapperServiceTestCase() {
             @Override
             protected Collection<? extends Plugin> getPlugins() {
                 return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin());
             }
-        }.createMapperService(mapping);
+        }.createMapperService(mappingXContent);
 
         var document = XContentBuilder.builder(XContentType.JSON.xContent());
-        dataGenerator.generateDocument(document);
+        document.map(documentGenerator.generate(template, mapping));
 
         mappingService.documentMapper().parse(new SourceToParse("1", BytesReference.bytes(document), XContentType.JSON));
     }
@@ -187,19 +165,17 @@ public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectAr
             @Override
             public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
                 return new DataSourceResponse.FieldTypeGenerator(
-                    () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG, false)
+                    () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG)
                 );
             }
         };
-
-        var dataGenerator = new DataGenerator(
-            DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).withMaxObjectDepth(2).build()
-        );
-
-        var mapping = XContentBuilder.builder(XContentType.JSON.xContent());
-        dataGenerator.writeMapping(mapping);
-
-        var document = XContentBuilder.builder(XContentType.JSON.xContent());
-        dataGenerator.generateDocument(document);
+        var specification = DataGeneratorSpecification.builder()
+            .withDataSourceHandlers(List.of(dataSourceOverride))
+            .withMaxObjectDepth(2)
+            .build();
+
+        var template = new TemplateGenerator(specification).generate();
+        var mapping = new MappingGenerator(specification).generate(template);
+        var ignored = new DocumentGenerator(specification).generate(template, mapping);
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/TemplateGeneratorTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/TemplateGeneratorTests.java
new file mode 100644
index 0000000000000..3f57addcac51e
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/TemplateGeneratorTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+
+public class TemplateGeneratorTests extends ESTestCase {
+    public void testSanity() {
+        var specification = DataGeneratorSpecification.buildDefault();
+        var generator = new TemplateGenerator(specification);
+
+        var template = generator.generate();
+        validateMappingTemplate(template.template());
+    }
+
+    private void validateMappingTemplate(Map<String, Template.Entry> template) {
+        // Just a high level sanity check, we test that mapping and documents make sense in DataGenerationTests.
+        for (var entry : template.entrySet()) {
+            assertNotNull(entry.getKey());
+            assertFalse(entry.getKey().isEmpty());
+            switch (entry.getValue()) {
+                case Template.Leaf leaf -> {
+                    assertEquals(entry.getKey(), leaf.name());
+                    assertNotNull(leaf.type());
+                }
+                case Template.Object object -> {
+                    assertEquals(entry.getKey(), object.name());
+                    assertNotNull(object.children());
+                    validateMappingTemplate(object.children());
+                }
+            }
+        }
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/LongGCDisruptionTests.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/LongGCDisruptionTests.java
deleted file mode 100644
index 72ecba8d502f1..0000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/disruption/LongGCDisruptionTests.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-package org.elasticsearch.test.disruption;
-
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
-
-import java.lang.management.ThreadInfo;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.regex.Pattern;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-
-public class LongGCDisruptionTests extends ESTestCase {
-
-    static class LockedExecutor {
-        ReentrantLock lock = new ReentrantLock();
-
-        public void executeLocked(Runnable r) {
-            lock.lock();
-            try {
-                r.run();
-            } finally {
-                lock.unlock();
-            }
-        }
-    }
-
-    @BeforeClass
-    public static void ignoreJdk20Plus() {
-        assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20);
-    }
-
-    public void testBlockingTimeout() throws Exception {
-        final String nodeName = "test_node";
-        LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) {
-            @Override
-            protected Pattern[] getUnsafeClasses() {
-                return new Pattern[] { Pattern.compile(LockedExecutor.class.getSimpleName()) };
-            }
-
-            @Override
-            protected long getSuspendingTimeoutInMillis() {
-                return 100;
-            }
-        };
-        final AtomicBoolean stop = new AtomicBoolean();
-        final CountDownLatch underLock = new CountDownLatch(1);
-        final CountDownLatch pauseUnderLock = new CountDownLatch(1);
-        final LockedExecutor lockedExecutor = new LockedExecutor();
-        final AtomicLong ops = new AtomicLong();
-        final Thread[] threads = new Thread[10];
-        try {
-            for (int i = 0; i < 10; i++) {
-                // at least one locked and one none lock thread
-                final boolean lockedExec = (i < 9 && randomBoolean()) || i == 0;
-                threads[i] = new Thread(() -> {
-                    while (stop.get() == false) {
-                        if (lockedExec) {
-                            lockedExecutor.executeLocked(() -> {
-                                try {
-                                    underLock.countDown();
-                                    ops.incrementAndGet();
-                                    pauseUnderLock.await();
-                                } catch (InterruptedException e) {
-
-                                }
-                            });
-                        } else {
-                            ops.incrementAndGet();
-                        }
-                    }
-                });
-                threads[i].setName("[" + nodeName + "][" + i + "]");
-                threads[i].start();
-            }
-            // make sure some threads are under lock
-            underLock.await();
-            RuntimeException e = expectThrows(RuntimeException.class, disruption::startDisrupting);
-            assertThat(e.getMessage(), containsString("suspending node threads took too long"));
-        } finally {
-            stop.set(true);
-            pauseUnderLock.countDown();
-            for (final Thread thread : threads) {
-                thread.join();
-            }
-        }
-    }
-
-    /**
-     * Checks that a GC disruption never blocks threads while they are doing something "unsafe"
-     * but does keep retrying until all threads can be safely paused
-     */
-    public void testNotBlockingUnsafeStackTraces() throws Exception {
-        final String nodeName = "test_node";
-        LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) {
-            @Override
-            protected Pattern[] getUnsafeClasses() {
-                return new Pattern[] { Pattern.compile(LockedExecutor.class.getSimpleName()) };
-            }
-        };
-        final AtomicBoolean stop = new AtomicBoolean();
-        final LockedExecutor lockedExecutor = new LockedExecutor();
-        final AtomicLong ops = new AtomicLong();
-        final Thread[] threads = new Thread[5];
-        final Runnable yieldAndIncrement = () -> {
-            Thread.yield(); // give some chance to catch this stack trace
-            ops.incrementAndGet();
-        };
-        try {
-            for (int i = 0; i < threads.length; i++) {
-                threads[i] = new Thread(() -> {
-                    for (int iter = 0; stop.get() == false; iter++) {
-                        if (iter % 2 == 0) {
-                            lockedExecutor.executeLocked(yieldAndIncrement);
-                        } else {
-                            yieldAndIncrement.run();
-                        }
-                    }
-                });
-                threads[i].setName("[" + nodeName + "][" + i + "]");
-                threads[i].start();
-            }
-            // make sure some threads are under lock
-            try {
-                disruption.startDisrupting();
-            } catch (RuntimeException e) {
-                if (e.getMessage().contains("suspending node threads took too long") && disruption.sawSlowSuspendBug()) {
-                    return;
-                }
-                throw new AssertionError(e);
-            }
-            long first = ops.get();
-            assertThat(lockedExecutor.lock.isLocked(), equalTo(false)); // no threads should own the lock
-            Thread.sleep(100);
-            assertThat(ops.get(), equalTo(first));
-            disruption.stopDisrupting();
-            assertBusy(() -> assertThat(ops.get(), greaterThan(first)));
-        } finally {
-            disruption.stopDisrupting();
-            stop.set(true);
-            for (final Thread thread : threads) {
-                thread.join();
-            }
-        }
-    }
-
-    public void testBlockDetection() throws Exception {
-        final String disruptedNodeName = "disrupted_node";
-        final String blockedNodeName = "blocked_node";
-        CountDownLatch waitForBlockDetectionResult = new CountDownLatch(1);
-        AtomicReference<ThreadInfo> blockDetectionResult = new AtomicReference<>();
-        LongGCDisruption disruption = new LongGCDisruption(random(), disruptedNodeName) {
-            @Override
-            protected Pattern[] getUnsafeClasses() {
-                return new Pattern[0];
-            }
-
-            @Override
-            protected void onBlockDetected(ThreadInfo blockedThread, @Nullable ThreadInfo blockingThread) {
-                blockDetectionResult.set(blockedThread);
-                waitForBlockDetectionResult.countDown();
-            }
-
-            @Override
-            protected long getBlockDetectionIntervalInMillis() {
-                return 10L;
-            }
-        };
-        if (disruption.isBlockDetectionSupported() == false) {
-            return;
-        }
-        final AtomicBoolean stop = new AtomicBoolean();
-        final CountDownLatch underLock = new CountDownLatch(1);
-        final CountDownLatch pauseUnderLock = new CountDownLatch(1);
-        final LockedExecutor lockedExecutor = new LockedExecutor();
-        final AtomicLong ops = new AtomicLong();
-        final List<Thread> threads = new ArrayList<>();
-        try {
-            for (int i = 0; i < 5; i++) {
-                // at least one locked and one none lock thread
-                final boolean lockedExec = (i < 4 && randomBoolean()) || i == 0;
-                Thread thread = new Thread(() -> {
-                    while (stop.get() == false) {
-                        if (lockedExec) {
-                            lockedExecutor.executeLocked(() -> {
-                                try {
-                                    underLock.countDown();
-                                    ops.incrementAndGet();
-                                    pauseUnderLock.await();
-                                } catch (InterruptedException e) {
-
-                                }
-                            });
-                        } else {
-                            ops.incrementAndGet();
-                        }
-                    }
-                });
-
-                thread.setName("[" + disruptedNodeName + "][" + i + "]");
-                threads.add(thread);
-                thread.start();
-            }
-
-            for (int i = 0; i < 5; i++) {
-                // at least one locked and one none lock thread
-                final boolean lockedExec = (i < 4 && randomBoolean()) || i == 0;
-                Thread thread = new Thread(() -> {
-                    while (stop.get() == false) {
-                        if (lockedExec) {
-                            lockedExecutor.executeLocked(() -> { ops.incrementAndGet(); });
-                        } else {
-                            ops.incrementAndGet();
-                        }
-                    }
-                });
-                thread.setName("[" + blockedNodeName + "][" + i + "]");
-                threads.add(thread);
-                thread.start();
-            }
-            // make sure some threads of test_node are under lock
-            underLock.await();
-            disruption.startDisrupting();
-            assertTrue(waitForBlockDetectionResult.await(30, TimeUnit.SECONDS));
-            disruption.stopDisrupting();
-
-            ThreadInfo threadInfo = blockDetectionResult.get();
-            assertNotNull(threadInfo);
-            assertThat(threadInfo.getThreadName(), containsString("[" + blockedNodeName + "]"));
-            assertThat(threadInfo.getLockOwnerName(), containsString("[" + disruptedNodeName + "]"));
-            assertThat(threadInfo.getLockInfo().getClassName(), containsString(ReentrantLock.class.getName()));
-        } finally {
-            stop.set(true);
-            pauseUnderLock.countDown();
-            for (final Thread thread : threads) {
-                thread.join();
-            }
-        }
-    }
-}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
index aa72d3248812e..3000819066495 100644
--- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
@@ -18,8 +18,10 @@
 public enum FeatureFlag {
     TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null),
     FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null),
+    SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null),
     CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null),
-    INFERENCE_DEFAULT_ELSER("es.inference_default_elser_feature_flag_enabled=true", Version.fromString("8.16.0"), null);
+    INFERENCE_DEFAULT_ELSER("es.inference_default_elser_feature_flag_enabled=true", Version.fromString("8.16.0"), null),
+    ML_SCALE_FROM_ZERO("es.ml_scale_from_zero_feature_flag_enabled=true", Version.fromString("8.16.0"), null);
 
     public final String systemProperty;
     public final Version from;
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java
index 885e02a8b5e6a..753ce8283afca 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java
@@ -69,8 +69,12 @@ public void testSimple() throws IOException {
         };
         AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"), dimensionField("dim"));
         testCase(iw -> {
-            iw.addDocuments(docs(1000, "1", 15, 37, 60, /*reset*/ 14));
-            iw.addDocuments(docs(1000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40));
+            for (Document document : docs(1000, "1", 15, 37, 60, /*reset*/ 14)) {
+                iw.addDocument(document);
+            }
+            for (Document document : docs(1000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40)) {
+                iw.addDocument(document);
+            }
         }, verifier, aggTestConfig);
     }
 
@@ -109,8 +113,12 @@ public void testNestedWithinDateHistogram() throws IOException {
         AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"), dimensionField("dim"))
             .withSplitLeavesIntoSeperateAggregators(false);
         testCase(iw -> {
-            iw.addDocuments(docs(2000, "1", 15, 37, 60, /*reset*/ 14));
-            iw.addDocuments(docs(2000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40));
+            for (Document document : docs(2000, "1", 15, 37, 60, /*reset*/ 14)) {
+                iw.addDocument(document);
+            }
+            for (Document document : docs(2000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40)) {
+                iw.addDocument(document);
+            }
         }, verifier, aggTestConfig);
     }
 
@@ -148,8 +156,12 @@ public void testNestedWithinAutoDateHistogram() throws IOException {
         AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"))
             .withSplitLeavesIntoSeperateAggregators(false);
         expectThrows(IllegalArgumentException.class, () -> testCase(iw -> {
-            iw.addDocuments(docs(2000, "1", 15, 37, 60, /*reset*/ 14));
-            iw.addDocuments(docs(2000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40));
+            for (Document document : docs(2000, "1", 15, 37, 60, /*reset*/ 14)) {
+                iw.addDocument(document);
+            }
+            for (Document document : docs(2000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40)) {
+                iw.addDocument(document);
+            }
         }, verifier, aggTestConfig));
     }
 
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java
index 20da254657c1a..04f0563e433a2 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java
@@ -170,7 +170,7 @@ public void testNoMatchingField() throws IOException {
     }
 
     public void testQueryFiltering() throws IOException {
-        testAggregation(new TermInSetQuery("text", new BytesRef("test0"), new BytesRef("test1")), iw -> {
+        testAggregation(new TermInSetQuery("text", List.of(new BytesRef("test0"), new BytesRef("test1"))), iw -> {
             for (int i = 0; i < 10; i++) {
                 iw.addDocument(singleton(new TextField("text", "test" + i, Field.Store.NO)));
             }
diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
index 52ecc40c957b7..aee344777779b 100644
--- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
+++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
@@ -314,9 +314,9 @@ private AsyncSearchResponse doNext() throws Exception {
                     assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures));
                     assertNull(newResponse.getSearchResponse().getAggregations());
                     assertNotNull(newResponse.getSearchResponse().getHits().getTotalHits());
-                    assertThat(newResponse.getSearchResponse().getHits().getTotalHits().value, equalTo(0L));
+                    assertThat(newResponse.getSearchResponse().getHits().getTotalHits().value(), equalTo(0L));
                     assertThat(
-                        newResponse.getSearchResponse().getHits().getTotalHits().relation,
+                        newResponse.getSearchResponse().getHits().getTotalHits().relation(),
                         equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)
                     );
                 } else {
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java
index 302bb68af6c61..fd4463df07a73 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java
@@ -62,7 +62,7 @@ public void testFetchFailuresAllShards() throws Exception {
             assertEquals(10, searchResponse.getSuccessfulShards());
             assertEquals(0, searchResponse.getFailedShards());
             assertEquals(0, searchResponse.getShardFailures().length);
-            assertEquals(10, searchResponse.getHits().getTotalHits().value);
+            assertEquals(10, searchResponse.getHits().getTotalHits().value());
             assertEquals(0, searchResponse.getHits().getHits().length);
             StringTerms terms = searchResponse.getAggregations().get("text");
             assertEquals(1, terms.getBuckets().size());
@@ -106,7 +106,7 @@ public void testFetchFailuresOnlySomeShards() throws Exception {
             assertEquals(10, searchResponse.getTotalShards());
             assertEquals(5, searchResponse.getSuccessfulShards());
             assertEquals(5, searchResponse.getFailedShards());
-            assertEquals(10, searchResponse.getHits().getTotalHits().value);
+            assertEquals(10, searchResponse.getHits().getTotalHits().value());
             assertEquals(5, searchResponse.getHits().getHits().length);
             StringTerms terms = searchResponse.getAggregations().get("text");
             assertEquals(1, terms.getBuckets().size());
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityAction.java
index 81b53487f9f15..836c8b155cb09 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityAction.java
@@ -47,7 +47,7 @@ public Request(TimeValue masterNodeTimeout) {
 
         public Request(final StreamInput in) throws IOException {
             super(in);
-            if (in.getTransportVersion().before(TransportVersions.GET_AUTOSCALING_CAPACITY_UNUSED_TIMEOUT)) {
+            if (in.getTransportVersion().before(TransportVersions.V_8_15_0)) {
                 in.readTimeValue(); // unused
             }
         }
@@ -55,7 +55,7 @@ public Request(final StreamInput in) throws IOException {
         @Override
         public void writeTo(final StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getTransportVersion().before(TransportVersions.GET_AUTOSCALING_CAPACITY_UNUSED_TIMEOUT)) {
+            if (out.getTransportVersion().before(TransportVersions.V_8_15_0)) {
                 out.writeTimeValue(AcknowledgedRequest.DEFAULT_ACK_TIMEOUT); // unused
             }
         }
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java
index 95b2324d03b52..7e7e954d1fa72 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java
@@ -175,7 +175,7 @@ public final int readVInt() throws IOException {
 
     @Override
     public final long readVLong() throws IOException {
-        if (9 <= buffer.remaining()) {
+        if (10 <= buffer.remaining()) {
             return ByteBufferStreamInput.readVLong(buffer);
         } else {
             return super.readVLong();
@@ -390,12 +390,11 @@ public IndexInput clone() {
 
     /** Returns default buffer sizes for the given {@link IOContext} */
     public static int bufferSize(IOContext context) {
-        switch (context.context) {
+        switch (context.context()) {
             case MERGE:
                 return MERGE_BUFFER_SIZE;
             case DEFAULT:
             case FLUSH:
-            case READ:
             default:
                 return BUFFER_SIZE;
         }
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index 3e5aaea43a9b9..8297ef5161fb0 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -82,7 +82,6 @@ tasks.named("precommit").configure {
 
 tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility")
-  task.skipTest("wildcard/30_ignore_above_synthetic_source/wildcard field type ignore_above", "Temporary until backported")
   task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs")
   task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.")
 })
diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
index 6f66e7e386066..9303191fcf75f 100644
--- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
+++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.WarningFailureException;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
@@ -1120,6 +1121,117 @@ public void testAutoFollowSearchableSnapshotsFails() throws Exception {
         }
     }
 
+    public void testNoWarningOnPromoteDatastreamWhenTemplateExistsOnFollower() throws Exception {
+        if ("follow".equals(targetCluster) == false) {
+            return;
+        }
+        testDataStreamPromotionWarnings(true);
+    }
+
+    public void testWarningOnPromoteDatastreamWhenTemplateDoesNotExistsOnFollower() {
+        if ("follow".equals(targetCluster) == false) {
+            return;
+        }
+        WarningFailureException exception = assertThrows(WarningFailureException.class, () -> testDataStreamPromotionWarnings(false));
+        assertThat(
+            exception.getMessage(),
+            containsString(
+                "does not have a matching index template. This will cause rollover to fail until a matching index template is created]"
+            )
+        );
+    }
+
+    private void testDataStreamPromotionWarnings(Boolean createFollowerTemplate) throws Exception {
+        final int numDocs = 64;
+        final String dataStreamName = getTestName().toLowerCase(Locale.ROOT) + "-dopromo";
+        final String autoFollowPatternName = getTestName().toLowerCase(Locale.ROOT);
+
+        int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices();
+        List<String> backingIndexNames = null;
+        try {
+            // Create index template
+            Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/" + getTestName().toLowerCase(Locale.ROOT));
+            putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\":[\"" + dataStreamName + "*\"],\"data_stream\":{}}");
+
+            if (createFollowerTemplate) {
+                assertOK(client().performRequest(putComposableIndexTemplateRequest));
+            }
+
+            // Create auto follow pattern
+            createAutoFollowPattern(client(), autoFollowPatternName, dataStreamName + "*", "leader_cluster", null);
+
+            // Create data stream and ensure that it is auto followed
+            try (var leaderClient = buildLeaderClient()) {
+                assertOK(leaderClient.performRequest(putComposableIndexTemplateRequest));
+
+                for (int i = 0; i < numDocs; i++) {
+                    var indexRequest = new Request("POST", "/" + dataStreamName + "/_doc");
+                    indexRequest.addParameter("refresh", "true");
+                    indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}");
+                    assertOK(leaderClient.performRequest(indexRequest));
+                }
+                verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1));
+                verifyDocuments(leaderClient, dataStreamName, numDocs);
+            }
+            assertBusy(() -> {
+                assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1));
+                verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1));
+                ensureYellow(dataStreamName);
+                verifyDocuments(client(), dataStreamName, numDocs);
+            });
+
+            // Rollover in leader cluster and ensure second backing index is replicated:
+            try (var leaderClient = buildLeaderClient()) {
+                var rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover");
+                assertOK(leaderClient.performRequest(rolloverRequest));
+                verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2));
+
+                var indexRequest = new Request("POST", "/" + dataStreamName + "/_doc");
+                indexRequest.addParameter("refresh", "true");
+                indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}");
+                assertOK(leaderClient.performRequest(indexRequest));
+                verifyDocuments(leaderClient, dataStreamName, numDocs + 1);
+            }
+            assertBusy(() -> {
+                assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 2));
+                verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2));
+                ensureYellow(dataStreamName);
+                verifyDocuments(client(), dataStreamName, numDocs + 1);
+            });
+
+            backingIndexNames = verifyDataStream(
+                client(),
+                dataStreamName,
+                backingIndexName(dataStreamName, 1),
+                backingIndexName(dataStreamName, 2)
+            );
+
+            // Promote local data stream
+            var promoteRequest = new Request("POST", "/_data_stream/_promote/" + dataStreamName);
+            Response response = client().performRequest(promoteRequest);
+            assertOK(response);
+        } finally {
+            if (backingIndexNames == null) {
+                // we failed to compute the actual backing index names in the test because we failed earlier on, guessing them on a
+                // best-effort basis
+                backingIndexNames = List.of(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2));
+            }
+
+            // These cleanup methods are copied from the finally block of other Data Stream tests in this class however
+            // they may no longer be required but have been included for completeness
+            cleanUpFollower(backingIndexNames, List.of(dataStreamName), List.of(autoFollowPatternName));
+            cleanUpLeader(backingIndexNames.subList(0, 1), List.of(dataStreamName), List.of());
+            Request deleteTemplateRequest = new Request("DELETE", "/_index_template/" + getTestName().toLowerCase(Locale.ROOT));
+            if (createFollowerTemplate) {
+                assertOK(client().performRequest(deleteTemplateRequest));
+            }
+            try (var leaderClient = buildLeaderClient()) {
+                assertOK(leaderClient.performRequest(deleteTemplateRequest));
+            }
+        }
+
+    }
+
     private int getNumberOfSuccessfulFollowedIndices() throws IOException {
         return getNumberOfSuccessfulFollowedIndices(client());
     }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java
index 2c633a43264f6..af2e99b047a71 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java
@@ -479,8 +479,7 @@ public <Request extends ActionRequest, Response extends TransportResponse> void
                 ActionListener<Response> listener
             ) {
                 final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(false);
-                try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-                    threadContext.markAsSystemContext();
+                try (var ignore = threadContext.newEmptySystemContext()) {
                     delegate.execute(connection, action, request, new ContextPreservingActionListener<>(supplier, listener));
                 }
             }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
index addf47662276d..e6857ba985306 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
@@ -195,9 +195,8 @@ private void removeRetentionLeaseForShard(
                     threadContext.newRestorableContext(true),
                     listener
                 );
-                try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) {
+                try (var ignore = threadPool.getThreadContext().newEmptySystemContext()) {
                     // we have to execute under the system context so that if security is enabled the removal is authorized
-                    threadContext.markAsSystemContext();
                     CcrRetentionLeases.asyncRemoveRetentionLease(leaderShardId, retentionLeaseId, remoteClient, preservedListener);
                 }
             }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
index ad42f549b1a80..c8136002bbd52 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
@@ -47,7 +47,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
@@ -447,10 +446,8 @@ public void restoreShard(
             // schedule renewals to run during the restore
             final Scheduler.Cancellable renewable = threadPool.scheduleWithFixedDelay(() -> {
                 logger.trace("{} background renewal of retention lease [{}] during restore", shardId, retentionLeaseId);
-                final ThreadContext threadContext = threadPool.getThreadContext();
-                try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                try (var ignore = threadPool.getThreadContext().newEmptySystemContext()) {
                     // we have to execute under the system context so that if security is enabled the renewal is authorized
-                    threadContext.markAsSystemContext();
                     CcrRetentionLeases.asyncRenewRetentionLease(
                         leaderShardId,
                         retentionLeaseId,
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java
index 6b390ab5747a8..164e6ed5406ae 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java
@@ -245,7 +245,7 @@ private Store.MetadataSnapshot getMetadata() throws IOException {
 
         private long readFileBytes(String fileName, ByteArray reference) throws IOException {
             try (Releasable ignored = keyedLock.acquire(fileName)) {
-                var context = fileName.startsWith(IndexFileNames.SEGMENTS) ? IOContext.READONCE : IOContext.READ;
+                var context = fileName.startsWith(IndexFileNames.SEGMENTS) ? IOContext.READONCE : IOContext.DEFAULT;
                 final IndexInput indexInput = cachedInputs.computeIfAbsent(fileName, f -> {
                     try {
                         return commitRef.getIndexCommit().getDirectory().openInput(fileName, context);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index 1b7875e4a36b4..618489abd687e 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -703,7 +703,7 @@ protected void atLeastDocsIndexed(Client client, String index, long numDocsRepli
             request.source(new SearchSourceBuilder().size(0));
             assertResponse(client.search(request), response -> {
                 assertNotNull(response.getHits().getTotalHits());
-                assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(numDocsReplicated));
+                assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(numDocsReplicated));
             });
         }, 60, TimeUnit.SECONDS);
     }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
index b4be0b33a464e..ef03fd0ba6f0e 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ccr.Ccr;
 import org.elasticsearch.xpack.ccr.CcrSettings;
@@ -332,8 +333,10 @@ public void testDynamicIndexSettingsAreClassified() {
         replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);
         replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME);
         replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING);
+        replicatedSettings.add(IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING);
         replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
         replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
+        replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
 
         for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
             // removed settings have no effect, they are only there for BWC
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
index 478a0d08d6612..150eddf039cec 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
@@ -700,6 +700,9 @@ public void testProcessOnceOnPrimary() throws Exception {
             case LOGSDB:
                 settingsBuilder.put("index.mode", IndexMode.LOGSDB.getName());
                 break;
+            case LOOKUP:
+                settingsBuilder.put("index.mode", IndexMode.LOOKUP.getName());
+                break;
             default:
                 throw new UnsupportedOperationException("Unknown index mode [" + indexMode + "]");
         }
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
index 0fea3c0d3b74f..1bf52b663b30f 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
@@ -274,7 +274,7 @@ private void assertHits(String index, int numDocsExpected, boolean sourceHadDele
         };
         assertResponse(prepareSearch(index).addSort(SeqNoFieldMapper.NAME, SortOrder.ASC).setSize(numDocsExpected), searchResponse -> {
             assertConsumer.accept(searchResponse, sourceHadDeletions);
-            assertEquals(numDocsExpected, searchResponse.getHits().getTotalHits().value);
+            assertEquals(numDocsExpected, searchResponse.getHits().getTotalHits().value());
         });
         SearchResponse searchResponse = prepareSearch(index).addSort(SeqNoFieldMapper.NAME, SortOrder.ASC)
             .setScroll(TimeValue.timeValueMinutes(1))
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java
index e66d41d089437..12864dd66a857 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java
@@ -9,9 +9,9 @@
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexWriter;
@@ -23,7 +23,6 @@
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
@@ -214,6 +213,11 @@ public NumericDocValues getNormValues(String field) {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public FloatVectorValues getFloatVectorValues(String field) throws IOException {
             throw new UnsupportedOperationException();
@@ -257,11 +261,6 @@ public LeafMetaData getMetaData() {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public Fields getTermVectors(int docId) {
-            throw new UnsupportedOperationException();
-        }
-
         @Override
         public TermVectors termVectors() throws IOException {
             throw new UnsupportedOperationException();
@@ -282,11 +281,6 @@ public int maxDoc() {
             return maxDoc;
         }
 
-        @Override
-        public void document(int docID, StoredFieldVisitor visitor) {
-            throw new UnsupportedOperationException();
-        }
-
         @Override
         protected void doClose() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java
index 2b01f4d7fa2a4..f280dcf9b3edf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -145,6 +146,7 @@ static boolean isEnterprise(String typeName) {
      * XContent param name to map the "enterprise" license type to "platinum"
      * for backwards compatibility with older clients
      */
+    @UpdateForV9(owner = UpdateForV9.Owner.SECURITY) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
     public static final String XCONTENT_HIDE_ENTERPRISE = "hide_enterprise";
 
     public static final Comparator<License> LATEST_ISSUE_DATE_FIRST = Comparator.comparing(License::issueDate).reversed();
@@ -524,13 +526,13 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t
         if (licenseVersion == VERSION_START) {
             builder.field(Fields.SUBSCRIPTION_TYPE, subscriptionType);
         }
-        builder.timeField(Fields.ISSUE_DATE_IN_MILLIS, Fields.ISSUE_DATE, issueDate);
+        builder.timestampFieldsFromUnixEpochMillis(Fields.ISSUE_DATE_IN_MILLIS, Fields.ISSUE_DATE, issueDate);
         if (licenseVersion == VERSION_START) {
             builder.field(Fields.FEATURE, feature);
         }
 
         if (expiryDate != LicenseSettings.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS) {
-            builder.timeField(Fields.EXPIRY_DATE_IN_MILLIS, Fields.EXPIRY_DATE, expiryDate);
+            builder.timestampFieldsFromUnixEpochMillis(Fields.EXPIRY_DATE_IN_MILLIS, Fields.EXPIRY_DATE, expiryDate);
         }
 
         if (licenseVersion >= VERSION_ENTERPRISE) {
@@ -551,7 +553,7 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t
             builder.humanReadable(previouslyHumanReadable);
         }
         if (licenseVersion >= VERSION_START_DATE) {
-            builder.timeField(Fields.START_DATE_IN_MILLIS, Fields.START_DATE, startDate);
+            builder.timestampFieldsFromUnixEpochMillis(Fields.START_DATE_IN_MILLIS, Fields.START_DATE, startDate);
         }
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java
index 01280b1d95f80..87c9713d12362 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java
@@ -214,10 +214,8 @@ private void remoteClusterLicense(final String clusterAlias, final ActionListene
             threadContext.newRestorableContext(false),
             listener
         );
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+        try (var ignore = threadContext.newEmptySystemContext()) {
             // we stash any context here since this is an internal execution and should not leak any existing context information
-            threadContext.markAsSystemContext();
-
             final XPackInfoRequest request = new XPackInfoRequest();
             request.setCategories(EnumSet.of(XPackInfoRequest.Category.LICENSE));
             try {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
index 24c081c237ee0..472249ed1ae3c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -27,7 +26,7 @@ public RestDeleteLicenseAction() {}
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(DELETE, "/_license").replaces(DELETE, "/_xpack/license", RestApiVersion.V_7).build());
+        return List.of(new Route(DELETE, "/_license"));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
index e0428c0ff2039..8dc8cf2395191 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestUtils;
@@ -24,9 +23,7 @@ public RestGetBasicStatus() {}
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_license/basic_status").replaces(GET, "/_xpack/license/basic_status", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_license/basic_status"));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
index 4240119df457d..92e46e3ea6564 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -27,9 +27,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.core.RestApiVersion.V_7;
-import static org.elasticsearch.core.RestApiVersion.V_8;
-import static org.elasticsearch.core.RestApiVersion.onOrAfter;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
@@ -43,7 +40,7 @@ public RestGetLicenseAction() {}
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(GET, "/_license").replaces(GET, "/_xpack/license", RestApiVersion.V_7).build());
+        return List.of(new Route(GET, "/_license"));
     }
 
     @Override
@@ -58,18 +55,12 @@ public String getName() {
      * The licenses are sorted by latest issue_date
      */
     @Override
+    @UpdateForV9(owner = UpdateForV9.Owner.SECURITY) // remove support for accept_enterprise param
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final Map<String, String> overrideParams = Maps.newMapWithExpectedSize(2);
         overrideParams.put(License.REST_VIEW_MODE, "true");
         overrideParams.put(License.LICENSE_VERSION_MODE, String.valueOf(License.VERSION_CURRENT));
 
-        if (request.getRestApiVersion() == V_7) {
-            // Hide enterprise licenses by default, there is an opt-in flag to show them
-            final boolean hideEnterprise = request.paramAsBoolean("accept_enterprise", false) == false;
-            final int licenseVersion = hideEnterprise ? License.VERSION_CRYPTO_ALGORITHMS : License.VERSION_CURRENT;
-            overrideParams.put(License.LICENSE_VERSION_MODE, String.valueOf(licenseVersion));
-            overrideParams.put(License.XCONTENT_HIDE_ENTERPRISE, String.valueOf(hideEnterprise));
-        }
         // In 7.x, there was an opt-in flag to show "enterprise" licenses. In 8.0 the flag is deprecated and can only be true
         // TODO Remove this from 9.0
         if (request.hasParam("accept_enterprise")) {
@@ -79,7 +70,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
                 "Including [accept_enterprise] in get license requests is deprecated."
                     + " The parameter will be removed in the next major version"
             );
-            if (request.paramAsBoolean("accept_enterprise", true) == false && request.getRestApiVersion().matches(onOrAfter(V_8))) {
+            if (request.paramAsBoolean("accept_enterprise", true) == false) {
                 throw new IllegalArgumentException("The [accept_enterprise] parameters may not be false");
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
index e42db16ded401..5cfde9288edec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestUtils;
@@ -24,9 +23,7 @@ public RestGetTrialStatus() {}
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_license/trial_status").replaces(GET, "/_xpack/license/trial_status", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_license/trial_status"));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
index 64556bcf69ecf..b44648966b936 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -26,9 +25,7 @@ public RestPostStartBasicLicense() {}
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_license/start_basic").replaces(POST, "/_xpack/license/start_basic", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_license/start_basic"));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
index 56fae76c2e2b8..c8321aa518237 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -28,9 +27,7 @@ public RestPostStartTrialLicense() {}
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_license/start_trial").replaces(POST, "/_xpack/license/start_trial", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_license/start_trial"));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
index 0798be6e53a14..cf24ac301046b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -28,10 +27,7 @@ public RestPutLicenseAction() {}
     @Override
     public List<Route> routes() {
         // TODO: remove POST endpoint?
-        return List.of(
-            Route.builder(POST, "/_license").replaces(POST, "/_xpack/license", RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_license").replaces(PUT, "/_xpack/license", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_license"), new Route(PUT, "/_license"));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
index 5ba0e584d63bb..b20f1a9d9ce23 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
@@ -14,8 +14,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.license.License;
 import org.elasticsearch.protocol.xpack.license.LicenseStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -206,27 +204,11 @@ public int hashCode() {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("uid", uid);
-
-            if (builder.getRestApiVersion() == RestApiVersion.V_7 && params.paramAsBoolean("accept_enterprise", false) == false) {
-                if (License.LicenseType.ENTERPRISE.getTypeName().equals(type)) {
-                    builder.field("type", License.LicenseType.PLATINUM.getTypeName());
-                } else {
-                    builder.field("type", type);
-                }
-
-                if (License.OperationMode.ENTERPRISE.description().equals(mode)) {
-                    builder.field("mode", License.OperationMode.PLATINUM.description());
-                } else {
-                    builder.field("mode", mode);
-                }
-            } else {
-                builder.field("type", type);
-                builder.field("mode", mode);
-            }
-
+            builder.field("type", type);
+            builder.field("mode", mode);
             builder.field("status", status.label());
             if (expiryDate != BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS) {
-                builder.timeField("expiry_date_in_millis", "expiry_date", expiryDate);
+                builder.timestampFieldsFromUnixEpochMillis("expiry_date_in_millis", "expiry_date", expiryDate);
             }
             return builder.endObject();
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java
index 093ec031d0b30..421a306babf29 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java
@@ -254,6 +254,7 @@ private SegmentCommitInfo syncSegment(
                             false,
                             IndexOptions.NONE,
                             DocValuesType.NONE,
+                            fieldInfo.docValuesSkipIndexType(),
                             -1,
                             fieldInfo.attributes(),
                             0,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java
index 527f8d1c176ec..0e185a90ed39b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java
@@ -10,10 +10,8 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
@@ -55,7 +53,7 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC
                 pattern = Strings.toLowercaseAscii(pattern);
             }
 
-            String tierPreference = getTierPreference(context);
+            String tierPreference = context.getTierPreference();
             if (tierPreference == null) {
                 return false;
             }
@@ -64,7 +62,7 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC
 
         @Override
         public Query existsQuery(SearchExecutionContext context) {
-            String tierPreference = getTierPreference(context);
+            String tierPreference = context.getTierPreference();
             if (tierPreference == null) {
                 return new MatchNoDocsQuery();
             }
@@ -77,26 +75,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            String tierPreference = getTierPreference(context);
+            String tierPreference = context.getTierPreference();
             return tierPreference == null ? ValueFetcher.EMPTY : ValueFetcher.singleton(tierPreference);
         }
-
-        /**
-         * Retrieve the first tier preference from the index setting. If the setting is not
-         * present, then return null.
-         */
-        private static String getTierPreference(QueryRewriteContext context) {
-            Settings settings = context.getIndexSettings().getSettings();
-            String value = DataTier.TIER_PREFERENCE_SETTING.get(settings);
-
-            if (Strings.hasText(value) == false) {
-                return null;
-            }
-
-            // Tier preference can be a comma-delimited list of tiers, ordered by preference
-            // It was decided we should only test the first of these potentially multiple preferences.
-            return value.split(",")[0].trim();
-        }
     }
 
     public DataTierFieldMapper() {
@@ -107,4 +88,5 @@ public DataTierFieldMapper() {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
index 4e7aa37fe1a0b..8c02462375e1d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -267,7 +267,7 @@ public static <T extends ActionResponse> T executeWithHeaders(
                 return supplier.get();
             }
         } else {
-            try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashContext()) {
+            try (var ignore = client.threadPool().getThreadContext().stashContext()) {
                 client.threadPool().getThreadContext().copyHeaders(filteredHeaders.entrySet());
                 return supplier.get();
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java
index dea158b425071..d315f09ebda88 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java
@@ -105,7 +105,7 @@ protected void searchResources(AbstractGetResourcesRequest request, TaskId paren
             listener.<SearchResponse>delegateFailure((l, response) -> {
                 List<Resource> docs = new ArrayList<>();
                 Set<String> foundResourceIds = new HashSet<>();
-                long totalHitCount = response.getHits().getTotalHits().value;
+                long totalHitCount = response.getHits().getTotalHits().value();
                 for (SearchHit hit : response.getHits().getHits()) {
                     try (
                         XContentParser parser = XContentHelper.createParserNotCompressed(
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
index 63fcd3dc4e798..e267c94e06892 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
@@ -91,8 +91,8 @@ public DataStreamStats(StreamInput in) throws IOException {
             this(
                 in.readVLong(),
                 in.readVLong(),
-                in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_TELEMETRY) ? in.readVLong() : 0,
-                in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_TELEMETRY) ? in.readVLong() : 0
+                in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readVLong() : 0,
+                in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readVLong() : 0
             );
         }
 
@@ -100,7 +100,7 @@ public DataStreamStats(StreamInput in) throws IOException {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(this.totalDataStreamCount);
             out.writeVLong(this.indicesBehindDataStream);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_TELEMETRY)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 out.writeVLong(this.failureStoreEnabledDataStreamCount);
                 out.writeVLong(this.failureStoreIndicesCount);
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
index fc9087d97bd79..0457de6edcc9f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
@@ -207,8 +207,8 @@ public CacheStats(StreamInput in) throws IOException {
                     in.readVLong(),
                     in.readVLong(),
                     in.readVLong(),
-                    in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_ADDITIONAL_STATS) ? in.readLong() : -1,
-                    in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_ADDITIONAL_STATS) ? in.readLong() : -1,
+                    in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readLong() : -1,
+                    in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readLong() : -1,
                     in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_STATS_SIZE_ADDED) ? in.readLong() : -1
                 );
             }
@@ -233,7 +233,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeVLong(hits);
                 out.writeVLong(misses);
                 out.writeVLong(evictions);
-                if (out.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_ADDITIONAL_STATS)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                     out.writeLong(hitsTimeInMillis);
                     out.writeLong(missesTimeInMillis);
                 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
index 9c679cd04c94d..33402671a2236 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
@@ -489,7 +489,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (managedByILM) {
             builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
             if (indexCreationDate != null) {
-                builder.timeField(
+                builder.timestampFieldsFromUnixEpochMillis(
                     INDEX_CREATION_DATE_MILLIS_FIELD.getPreferredName(),
                     INDEX_CREATION_DATE_FIELD.getPreferredName(),
                     indexCreationDate
@@ -500,26 +500,42 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 );
             }
             if (lifecycleDate != null) {
-                builder.timeField(LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), LIFECYCLE_DATE_FIELD.getPreferredName(), lifecycleDate);
+                builder.timestampFieldsFromUnixEpochMillis(
+                    LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(),
+                    LIFECYCLE_DATE_FIELD.getPreferredName(),
+                    lifecycleDate
+                );
                 builder.field(AGE_FIELD.getPreferredName(), getAge(nowSupplier).toHumanReadableString(2));
             }
             if (phase != null) {
                 builder.field(PHASE_FIELD.getPreferredName(), phase);
             }
             if (phaseTime != null) {
-                builder.timeField(PHASE_TIME_MILLIS_FIELD.getPreferredName(), PHASE_TIME_FIELD.getPreferredName(), phaseTime);
+                builder.timestampFieldsFromUnixEpochMillis(
+                    PHASE_TIME_MILLIS_FIELD.getPreferredName(),
+                    PHASE_TIME_FIELD.getPreferredName(),
+                    phaseTime
+                );
             }
             if (action != null) {
                 builder.field(ACTION_FIELD.getPreferredName(), action);
             }
             if (actionTime != null) {
-                builder.timeField(ACTION_TIME_MILLIS_FIELD.getPreferredName(), ACTION_TIME_FIELD.getPreferredName(), actionTime);
+                builder.timestampFieldsFromUnixEpochMillis(
+                    ACTION_TIME_MILLIS_FIELD.getPreferredName(),
+                    ACTION_TIME_FIELD.getPreferredName(),
+                    actionTime
+                );
             }
             if (step != null) {
                 builder.field(STEP_FIELD.getPreferredName(), step);
             }
             if (stepTime != null) {
-                builder.timeField(STEP_TIME_MILLIS_FIELD.getPreferredName(), STEP_TIME_FIELD.getPreferredName(), stepTime);
+                builder.timestampFieldsFromUnixEpochMillis(
+                    STEP_TIME_MILLIS_FIELD.getPreferredName(),
+                    STEP_TIME_FIELD.getPreferredName(),
+                    stepTime
+                );
             }
             if (Strings.hasLength(failedStep)) {
                 builder.field(FAILED_STEP_FIELD.getPreferredName(), failedStep);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
index 3674103eda215..f8cb371687d72 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -92,10 +93,13 @@ public OperationMode getOperationMode() {
     }
 
     public Map<String, LifecyclePolicy> getPolicies() {
-        return policyMetadatas.values()
-            .stream()
-            .map(LifecyclePolicyMetadata::getPolicy)
-            .collect(Collectors.toMap(LifecyclePolicy::getName, Function.identity()));
+        // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+        Map<String, LifecyclePolicy> policies = new HashMap<>(policyMetadatas.size());
+        for (LifecyclePolicyMetadata policyMetadata : policyMetadatas.values()) {
+            LifecyclePolicy policy = policyMetadata.getPolicy();
+            policies.put(policy.getName(), policy);
+        }
+        return Collections.unmodifiableMap(policies);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java
index 78ff08d5ced5b..2aed198d2e5fe 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java
@@ -130,7 +130,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(PHASE_DEFINITION_FIELD.getPreferredName(), phase);
         }
         builder.field(VERSION_FIELD.getPreferredName(), version);
-        builder.timeField(MODIFIED_DATE_IN_MILLIS_FIELD.getPreferredName(), "modified_date", modifiedDate);
+        builder.timestampFieldsFromUnixEpochMillis(MODIFIED_DATE_IN_MILLIS_FIELD.getPreferredName(), "modified_date", modifiedDate);
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
index 3e6c00eeadba4..3d140f5a9d764 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
@@ -126,9 +126,7 @@ public void performAction(
         RolloverRequest rolloverRequest = new RolloverRequest(rolloverTarget, null).masterNodeTimeout(TimeValue.MAX_VALUE);
         if (targetFailureStore) {
             rolloverRequest.setIndicesOptions(
-                IndicesOptions.builder(rolloverRequest.indicesOptions())
-                    .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
-                    .build()
+                IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
             );
         }
         // We don't wait for active shards when we perform the rollover because the
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
index 7b751994222b1..aa20e33a3fbf2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
@@ -247,9 +247,7 @@ RolloverRequest createRolloverRequest(
         rolloverRequest.setConditions(applyDefaultConditions(conditions, rolloverOnlyIfHasDocuments));
         if (targetFailureStore) {
             rolloverRequest.setIndicesOptions(
-                IndicesOptions.builder(rolloverRequest.indicesOptions())
-                    .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
-                    .build()
+                IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
             );
         }
         return rolloverRequest;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
index e9d612751e48f..226fe3630b387 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
@@ -50,7 +50,7 @@ public Request(StreamInput in) throws IOException {
             super(in);
             this.inferenceEndpointId = in.readString();
             this.taskType = TaskType.fromStream(in);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 this.forceDelete = Boolean.TRUE.equals(in.readOptionalBoolean());
                 this.dryRun = Boolean.TRUE.equals(in.readOptionalBoolean());
             } else {
@@ -80,7 +80,7 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(inferenceEndpointId);
             taskType.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 out.writeOptionalBoolean(forceDelete);
                 out.writeOptionalBoolean(dryRun);
             }
@@ -121,7 +121,7 @@ public Response(boolean acknowledged, Set<String> pipelineIds, Set<String> seman
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 pipelineIds = in.readCollectionAsSet(StreamInput::readString);
             } else {
                 pipelineIds = Set.of();
@@ -140,7 +140,7 @@ public Response(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 out.writeCollection(pipelineIds, StreamOutput::writeString);
             }
             if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS)) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java
index 5a779ada4e182..ea0462d0f103e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java
@@ -34,19 +34,41 @@ public GetInferenceModelAction() {
 
     public static class Request extends AcknowledgedRequest<GetInferenceModelAction.Request> {
 
+        private static boolean PERSIST_DEFAULT_CONFIGS = true;
+
         private final String inferenceEntityId;
         private final TaskType taskType;
+        // Default endpoint configurations are persisted on first read.
+        // Set to false to avoid persisting on read.
+        // This setting only applies to GET * requests. It has
+        // no effect when getting a single model
+        private final boolean persistDefaultConfig;
 
         public Request(String inferenceEntityId, TaskType taskType) {
             super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
             this.inferenceEntityId = Objects.requireNonNull(inferenceEntityId);
             this.taskType = Objects.requireNonNull(taskType);
+            this.persistDefaultConfig = PERSIST_DEFAULT_CONFIGS;
+        }
+
+        public Request(String inferenceEntityId, TaskType taskType, boolean persistDefaultConfig) {
+            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+            this.inferenceEntityId = Objects.requireNonNull(inferenceEntityId);
+            this.taskType = Objects.requireNonNull(taskType);
+            this.persistDefaultConfig = persistDefaultConfig;
         }
 
         public Request(StreamInput in) throws IOException {
             super(in);
             this.inferenceEntityId = in.readString();
             this.taskType = TaskType.fromStream(in);
+            if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ)
+                || in.getTransportVersion().isPatchFrom(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16)) {
+                this.persistDefaultConfig = in.readBoolean();
+            } else {
+                this.persistDefaultConfig = PERSIST_DEFAULT_CONFIGS;
+            }
+
         }
 
         public String getInferenceEntityId() {
@@ -57,11 +79,19 @@ public TaskType getTaskType() {
             return taskType;
         }
 
+        public boolean isPersistDefaultConfig() {
+            return persistDefaultConfig;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(inferenceEntityId);
             taskType.writeTo(out);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ)
+                || out.getTransportVersion().isPatchFrom(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16)) {
+                out.writeBoolean(this.persistDefaultConfig);
+            }
         }
 
         @Override
@@ -69,12 +99,14 @@ public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             Request request = (Request) o;
-            return Objects.equals(inferenceEntityId, request.inferenceEntityId) && taskType == request.taskType;
+            return Objects.equals(inferenceEntityId, request.inferenceEntityId)
+                && taskType == request.taskType
+                && persistDefaultConfig == request.persistDefaultConfig;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(inferenceEntityId, taskType);
+            return Objects.hash(inferenceEntityId, taskType, persistDefaultConfig);
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java
new file mode 100644
index 0000000000000..cc59ae890467b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java
@@ -0,0 +1,278 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.action;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.MlStrings;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.inference.ModelConfigurations.SERVICE_SETTINGS;
+import static org.elasticsearch.inference.ModelConfigurations.TASK_SETTINGS;
+
+public class UpdateInferenceModelAction extends ActionType<UpdateInferenceModelAction.Response> {
+
+    public static final UpdateInferenceModelAction INSTANCE = new UpdateInferenceModelAction();
+    public static final String NAME = "cluster:admin/xpack/inference/update";
+
+    public UpdateInferenceModelAction() {
+        super(NAME);
+    }
+
+    public record Settings(
+        @Nullable Map<String, Object> serviceSettings,
+        @Nullable Map<String, Object> taskSettings,
+        @Nullable TaskType taskType
+    ) {}
+
+    public static class Request extends AcknowledgedRequest<Request> {
+
+        private final String inferenceEntityId;
+        private final BytesReference content;
+        private final XContentType contentType;
+        private final TaskType taskType;
+        private Settings settings;
+
+        public Request(String inferenceEntityId, BytesReference content, XContentType contentType, TaskType taskType, TimeValue timeout) {
+            super(timeout, DEFAULT_ACK_TIMEOUT);
+            this.inferenceEntityId = inferenceEntityId;
+            this.content = content;
+            this.contentType = contentType;
+            this.taskType = taskType;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.inferenceEntityId = in.readString();
+            this.content = in.readBytesReference();
+            this.taskType = TaskType.fromStream(in);
+            this.contentType = in.readEnum(XContentType.class);
+        }
+
+        public String getInferenceEntityId() {
+            return inferenceEntityId;
+        }
+
+        public TaskType getTaskType() {
+            return taskType;
+        }
+
+        /**
+         * The body of the request.
+         * For in-cluster models, this is expected to contain some of the following:
+         * "number_of_allocations": `an integer`
+         *
+         * For third-party services, this is expected to contain:
+         *  "service_settings": {
+         *      "api_key": `a string` // service settings can only contain an api key
+         *  }
+         *  "task_settings": { a map of settings }
+         *
+         */
+        public BytesReference getContent() {
+            return content;
+        }
+
+        /**
+         * The body of the request as a map.
+         * The map is validated such that only allowed fields are present.
+         * If any fields in the body are not on the allow list, this function will throw an exception.
+         */
+        public Settings getContentAsSettings() {
+            if (settings == null) { // settings is deterministic on content, so we only need to compute it once
+                Map<String, Object> unvalidatedMap = XContentHelper.convertToMap(content, false, contentType).v2();
+                Map<String, Object> serviceSettings = new HashMap<>();
+                Map<String, Object> taskSettings = new HashMap<>();
+                TaskType taskType = null;
+
+                if (unvalidatedMap.isEmpty()) {
+                    throw new ElasticsearchStatusException("Request body is empty", RestStatus.BAD_REQUEST);
+                }
+
+                if (unvalidatedMap.containsKey("task_type")) {
+                    if (unvalidatedMap.get("task_type") instanceof String taskTypeString) {
+                        taskType = TaskType.fromStringOrStatusException(taskTypeString);
+                    } else {
+                        throw new ElasticsearchStatusException(
+                            "Failed to parse [task_type] in update request [{}]",
+                            RestStatus.INTERNAL_SERVER_ERROR,
+                            unvalidatedMap.toString()
+                        );
+                    }
+                    unvalidatedMap.remove("task_type");
+                }
+
+                if (unvalidatedMap.containsKey(SERVICE_SETTINGS)) {
+                    if (unvalidatedMap.get(SERVICE_SETTINGS) instanceof Map<?, ?> tempMap) {
+                        for (Map.Entry<?, ?> entry : (tempMap).entrySet()) {
+                            if (entry.getKey() instanceof String key && entry.getValue() instanceof Object value) {
+                                serviceSettings.put(key, value);
+                            } else {
+                                throw new ElasticsearchStatusException(
+                                    "Failed to parse update request [{}]",
+                                    RestStatus.INTERNAL_SERVER_ERROR,
+                                    unvalidatedMap.toString()
+                                );
+                            }
+                        }
+                        unvalidatedMap.remove(SERVICE_SETTINGS);
+                    } else {
+                        throw new ElasticsearchStatusException(
+                            "Unable to parse service settings in the request [{}]",
+                            RestStatus.BAD_REQUEST,
+                            unvalidatedMap.toString()
+                        );
+                    }
+                }
+
+                if (unvalidatedMap.containsKey(TASK_SETTINGS)) {
+                    if (unvalidatedMap.get(TASK_SETTINGS) instanceof Map<?, ?> tempMap) {
+                        for (Map.Entry<?, ?> entry : (tempMap).entrySet()) {
+                            if (entry.getKey() instanceof String key && entry.getValue() instanceof Object value) {
+                                taskSettings.put(key, value);
+                            } else {
+                                throw new ElasticsearchStatusException(
+                                    "Failed to parse update request [{}]",
+                                    RestStatus.INTERNAL_SERVER_ERROR,
+                                    unvalidatedMap.toString()
+                                );
+                            }
+                        }
+                        unvalidatedMap.remove(TASK_SETTINGS);
+                    } else {
+                        throw new ElasticsearchStatusException(
+                            "Unable to parse task settings in the request [{}]",
+                            RestStatus.BAD_REQUEST,
+                            unvalidatedMap.toString()
+                        );
+                    }
+                }
+
+                if (unvalidatedMap.isEmpty() == false) {
+                    throw new ElasticsearchStatusException(
+                        "Request contained fields which cannot be updated, remove these fields and try again [{}]",
+                        RestStatus.BAD_REQUEST,
+                        unvalidatedMap.toString()
+                    );
+                }
+
+                this.settings = new Settings(
+                    serviceSettings.isEmpty() == false ? Collections.unmodifiableMap(serviceSettings) : null,
+                    taskSettings.isEmpty() == false ? Collections.unmodifiableMap(taskSettings) : null,
+                    taskType
+                );
+            }
+            return this.settings;
+        }
+
+        public XContentType getContentType() {
+            return contentType;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(inferenceEntityId);
+            taskType.writeTo(out);
+            out.writeBytesReference(content);
+            XContentHelper.writeTo(out, contentType);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = new ActionRequestValidationException();
+            if (MlStrings.isValidId(this.inferenceEntityId) == false) {
+                validationException.addValidationError(Messages.getMessage(Messages.INVALID_ID, "inference_id", this.inferenceEntityId));
+            }
+
+            if (validationException.validationErrors().isEmpty() == false) {
+                return validationException;
+            } else {
+                return null;
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Objects.equals(inferenceEntityId, request.inferenceEntityId)
+                && Objects.equals(content, request.content)
+                && contentType == request.contentType
+                && taskType == request.taskType;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(inferenceEntityId, content, contentType, taskType);
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private final ModelConfigurations model;
+
+        public Response(ModelConfigurations model) {
+            this.model = model;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            model = new ModelConfigurations(in);
+        }
+
+        public ModelConfigurations getModel() {
+            return model;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            model.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return model.toFilteredXContent(builder, params);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response response = (Response) o;
+            return Objects.equals(model, response.model);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(model);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
index 902c69cef558e..5c63a60103139 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
@@ -50,7 +50,7 @@ public ChatCompletionResults(StreamInput in) throws IOException {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContentHelper.array(COMPLETION, results.iterator());
+        return ChunkedToXContent.builder(params).array(COMPLETION, results.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java
index 187b186fcd91d..c961050acefdb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.xcontent.ToXContent;
@@ -79,7 +79,7 @@ public List<MlChunkedTextExpansionResults.ChunkedResult> getChunkedResults() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContentHelper.array(FIELD_NAME, chunkedResults.iterator());
+        return ChunkedToXContent.builder(params).array(FIELD_NAME, chunkedResults.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java
index cc245c40c51e3..6bd66664068d5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.xcontent.ToXContent;
@@ -64,7 +64,7 @@ public InferenceChunkedTextEmbeddingByteResults(StreamInput in) throws IOExcepti
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContentHelper.array(FIELD_NAME, chunks.iterator());
+        return ChunkedToXContent.builder(params).array(FIELD_NAME, chunks.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java
index 4b4d77cd3f043..369f22a807913 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.xcontent.ToXContent;
@@ -77,7 +77,7 @@ public static InferenceChunkedTextEmbeddingFloatResults ofMlResults(MlChunkedTex
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         // TODO add isTruncated flag
-        return ChunkedToXContentHelper.array(FIELD_NAME, chunks.iterator());
+        return ChunkedToXContent.builder(params).array(FIELD_NAME, chunks.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
index 16dca7b04d526..c1be1ce265f6b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.xcontent.ToXContent;
@@ -62,7 +62,7 @@ public int getFirstEmbeddingSize() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContentHelper.array(TEXT_EMBEDDING_BYTES, embeddings.iterator());
+        return ChunkedToXContent.builder(params).array(TEXT_EMBEDDING_BYTES, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
index 9f9bdfec7cfae..8551250348928 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
@@ -103,7 +103,7 @@ public int getFirstEmbeddingSize() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContentHelper.array(TEXT_EMBEDDING, embeddings.iterator());
+        return ChunkedToXContent.builder(params).array(TEXT_EMBEDDING, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
index 6ebf15bf34937..9c764babe33fc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -31,8 +31,6 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.TransportVersions.ML_RERANK_DOC_OPTIONAL;
-
 public class RankedDocsResults implements InferenceServiceResults {
     public static final String NAME = "rerank_service_results";
     public static final String RERANK = TaskType.RERANK.toString();
@@ -114,7 +112,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         public static RankedDoc of(StreamInput in) throws IOException {
-            if (in.getTransportVersion().onOrAfter(ML_RERANK_DOC_OPTIONAL)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 return new RankedDoc(in.readInt(), in.readFloat(), in.readOptionalString());
             } else if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 return new RankedDoc(in.readInt(), in.readFloat(), in.readString());
@@ -125,7 +123,7 @@ public static RankedDoc of(StreamInput in) throws IOException {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().onOrAfter(ML_RERANK_DOC_OPTIONAL)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 out.writeInt(index);
                 out.writeFloat(relevanceScore);
                 out.writeOptionalString(text);
@@ -176,7 +174,7 @@ public List<RankedDoc> getRankedDocs() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContentHelper.array(RERANK, rankedDocs.iterator());
+        return ChunkedToXContent.builder(params).array(RERANK, rankedDocs.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
index dd8229c604ecb..318a292b47730 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
@@ -72,7 +72,7 @@ public static SparseEmbeddingResults of(List<? extends InferenceResults> results
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContentHelper.array(SPARSE_EMBEDDING, embeddings.iterator());
+        return ChunkedToXContent.builder(params).array(SPARSE_EMBEDDING, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java
index 3e61f6b4e9258..6c49cadb8d189 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java
@@ -37,6 +37,14 @@ public final class MachineLearningField {
         Setting.Property.NodeScope
     );
 
+    public static final Setting<Integer> MAX_LAZY_ML_NODES = Setting.intSetting(
+        "xpack.ml.max_lazy_ml_nodes",
+        0,
+        0,
+        Setting.Property.OperatorDynamic,
+        Setting.Property.NodeScope
+    );
+
     /**
      * This boolean value indicates if `max_machine_memory_percent` should be ignored and an automatic calculation is used instead.
      *
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java
index 381fb30b65e68..bddae0417e467 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -43,7 +44,10 @@ public static class Request extends BaseTasksRequest<Request> implements ToXCont
 
         public static final ParseField TIMEOUT = new ParseField("timeout");
         public static final ParseField FORCE = new ParseField("force");
+
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate forRestApiVersion
         public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match").forRestApiVersion(onOrAfter(RestApiVersion.V_8));
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         public static final ParseField ALLOW_NO_MATCH_V7 = new ParseField("allow_no_match", DEPRECATED_ALLOW_NO_JOBS_PARAM)
             .forRestApiVersion(equalTo(RestApiVersion.V_7));
         public static final ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
@@ -179,11 +183,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Job.ID.getPreferredName(), jobId);
             builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep());
             builder.field(FORCE.getPreferredName(), force);
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(DEPRECATED_ALLOW_NO_JOBS_PARAM, allowNoMatch);
-            } else {
-                builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch);
-            }
+            builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch);
             builder.endObject();
             return builder;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java
index 082f6d7aff899..72f05091c1ccd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java
@@ -255,7 +255,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.startObject();
             builder.field("flushed", flushed);
             if (lastFinalizedBucketEnd != null) {
-                builder.timeField(
+                builder.timestampFieldsFromUnixEpochMillis(
                     FlushAcknowledgement.LAST_FINALIZED_BUCKET_END.getPreferredName(),
                     FlushAcknowledgement.LAST_FINALIZED_BUCKET_END.getPreferredName() + "_string",
                     lastFinalizedBucketEnd.toEpochMilli()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java
index 12565ea49e058..47bc6df5f6536 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -67,7 +68,9 @@ public static class Request extends ActionRequest implements ToXContentObject {
         public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim");
         public static final ParseField START = new ParseField("start");
         public static final ParseField END = new ParseField("end");
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate forRestApiVersion
         public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match").forRestApiVersion(onOrAfter(RestApiVersion.V_8));
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         public static final ParseField ALLOW_NO_MATCH_V7 = new ParseField("allow_no_match", DEPRECATED_ALLOW_NO_JOBS_PARAM)
             .forRestApiVersion(equalTo(RestApiVersion.V_7));
 
@@ -243,11 +246,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (end != null) {
                 builder.field(END.getPreferredName(), String.valueOf(end));
             }
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(DEPRECATED_ALLOW_NO_JOBS_PARAM, allowNoMatch);
-            } else {
-                builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch);
-            }
+            builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch);
             builder.endObject();
             return builder;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java
index c61c62ed23a4d..8ad275e29b09a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.ingest.IngestStats;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -34,7 +33,6 @@
 import java.util.Objects;
 import java.util.Set;
 
-import static org.elasticsearch.core.RestApiVersion.onOrAfter;
 import static org.elasticsearch.core.Strings.format;
 
 public class GetTrainedModelsStatsAction extends ActionType<GetTrainedModelsStatsAction.Response> {
@@ -172,7 +170,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 if (this.inferenceStats != null) {
                     builder.field(INFERENCE_STATS.getPreferredName(), this.inferenceStats);
                 }
-                if (deploymentStats != null && builder.getRestApiVersion().matches(onOrAfter(RestApiVersion.V_8))) {
+                if (deploymentStats != null) {
                     builder.field(DEPLOYMENT_STATS.getPreferredName(), this.deploymentStats);
                 }
                 builder.endObject();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
index e6b580f62fdd3..05d9496e434c3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
@@ -198,7 +198,7 @@ public Request(StreamInput in) throws IOException {
             } else {
                 prefixType = TrainedModelPrefixStrings.PrefixType.NONE;
             }
-            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_CHUNK_INFERENCE_OPTION)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 chunked = in.readBoolean();
             } else {
                 chunked = false;
@@ -285,7 +285,7 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
                 out.writeEnum(prefixType);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_CHUNK_INFERENCE_OPTION)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 out.writeBoolean(chunked);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
index 34ebdcb7f9f9f..b298d486c9e03 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
@@ -71,7 +71,7 @@ public class StartTrainedModelDeploymentAction extends ActionType<CreateTrainedM
     public static final AllocationStatus.State DEFAULT_WAITFOR_STATE = AllocationStatus.State.STARTED;
     public static final int DEFAULT_NUM_ALLOCATIONS = 1;
     public static final int DEFAULT_NUM_THREADS = 1;
-    public static final int DEFAULT_QUEUE_CAPACITY = 1024;
+    public static final int DEFAULT_QUEUE_CAPACITY = 10_000;
     public static final Priority DEFAULT_PRIORITY = Priority.NORMAL;
 
     public StartTrainedModelDeploymentAction() {
@@ -89,7 +89,7 @@ public static class Request extends MasterNodeRequest<Request> implements ToXCon
         /**
          * If the queue is created then we can OOM when we create the queue.
          */
-        private static final int MAX_QUEUE_CAPACITY = 1_000_000;
+        private static final int MAX_QUEUE_CAPACITY = 100_000;
 
         public static final ParseField MODEL_ID = new ParseField("model_id");
         public static final ParseField DEPLOYMENT_ID = new ParseField("deployment_id");
@@ -623,6 +623,9 @@ public String getDeploymentId() {
          * @return the estimated memory (in bytes) required for the model deployment to run
          */
         public long estimateMemoryUsageBytes() {
+            if (numberOfAllocations == 0) {
+                return 0;
+            }
             // We already take into account 2x the model bytes. If the cache size is larger than the model bytes, then
             // we need to take it into account when returning the estimate.
             if (cacheSize != null && cacheSize.getBytes() > modelBytes) {
@@ -796,6 +799,9 @@ public static long estimateMemoryUsageBytes(
         long perAllocationMemoryBytes,
         int numberOfAllocations
     ) {
+        if (numberOfAllocations == 0) {
+            return 0;
+        }
         // While loading the model in the process we need twice the model size.
 
         // 1. If ELSER v1 or v2 then 2004MB
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
index 2fd00a5ea3983..bd4aac7ccad89 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -46,7 +47,9 @@ public static class Request extends BaseTasksRequest<Request> implements ToXCont
 
         public static final ParseField TIMEOUT = new ParseField("timeout");
         public static final ParseField FORCE = new ParseField("force");
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate forRestApiVersion
         public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match").forRestApiVersion(onOrAfter(RestApiVersion.V_8));
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         public static final ParseField ALLOW_NO_MATCH_V7 = new ParseField("allow_no_match", DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM)
             .forRestApiVersion(equalTo(RestApiVersion.V_7));
 
@@ -169,11 +172,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId);
             builder.field(TIMEOUT.getPreferredName(), stopTimeout.getStringRep());
             builder.field(FORCE.getPreferredName(), force);
-            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                builder.field(DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM, allowNoMatch);
-            } else {
-                builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch);
-            }
+            builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch);
             builder.endObject();
             return builder;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java
index d4da74df85ba9..2ea605753ccfc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java
@@ -332,17 +332,33 @@ public String getByFieldValue() {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(ANNOTATION.getPreferredName(), annotation);
-        builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            CREATE_TIME.getPreferredName(),
+            CREATE_TIME.getPreferredName() + "_string",
+            createTime.getTime()
+        );
         builder.field(CREATE_USERNAME.getPreferredName(), createUsername);
-        builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            TIMESTAMP.getPreferredName(),
+            TIMESTAMP.getPreferredName() + "_string",
+            timestamp.getTime()
+        );
         if (endTimestamp != null) {
-            builder.timeField(END_TIMESTAMP.getPreferredName(), END_TIMESTAMP.getPreferredName() + "_string", endTimestamp.getTime());
+            builder.timestampFieldsFromUnixEpochMillis(
+                END_TIMESTAMP.getPreferredName(),
+                END_TIMESTAMP.getPreferredName() + "_string",
+                endTimestamp.getTime()
+            );
         }
         if (jobId != null) {
             builder.field(Job.ID.getPreferredName(), jobId);
         }
         if (modifiedTime != null) {
-            builder.timeField(MODIFIED_TIME.getPreferredName(), MODIFIED_TIME.getPreferredName() + "_string", modifiedTime.getTime());
+            builder.timestampFieldsFromUnixEpochMillis(
+                MODIFIED_TIME.getPreferredName(),
+                MODIFIED_TIME.getPreferredName() + "_string",
+                modifiedTime.getTime()
+            );
         }
         if (modifiedUsername != null) {
             builder.field(MODIFIED_USERNAME.getPreferredName(), modifiedUsername);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java
index c6fa4e052c683..b007c1da451f5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java
@@ -217,8 +217,16 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(DESCRIPTION.getPreferredName(), description);
-        builder.timeField(START_TIME.getPreferredName(), START_TIME.getPreferredName() + "_string", startTime.toEpochMilli());
-        builder.timeField(END_TIME.getPreferredName(), END_TIME.getPreferredName() + "_string", endTime.toEpochMilli());
+        builder.timestampFieldsFromUnixEpochMillis(
+            START_TIME.getPreferredName(),
+            START_TIME.getPreferredName() + "_string",
+            startTime.toEpochMilli()
+        );
+        builder.timestampFieldsFromUnixEpochMillis(
+            END_TIME.getPreferredName(),
+            END_TIME.getPreferredName() + "_string",
+            endTime.toEpochMilli()
+        );
         builder.field(SKIP_RESULT.getPreferredName(), skipResult);
         builder.field(SKIP_MODEL_UPDATE.getPreferredName(), skipModelUpdate);
         if (forceTimeShift != null) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java
index 7a3334aad00f1..694d248efc7be 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java
@@ -30,8 +30,8 @@ public SearchInterval(StreamInput in) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.timeField(START_MS.getPreferredName(), START.getPreferredName(), startMs);
-        builder.timeField(END_MS.getPreferredName(), END.getPreferredName(), endMs);
+        builder.timestampFieldsFromUnixEpochMillis(START_MS.getPreferredName(), START.getPreferredName(), startMs);
+        builder.timestampFieldsFromUnixEpochMillis(END_MS.getPreferredName(), END.getPreferredName(), endMs);
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java
index 4c9028f64c2fd..779c6ef263ebe 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java
@@ -258,7 +258,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(ID.getPreferredName(), id);
         if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) {
             if (createTime != null) {
-                builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
+                builder.timestampFieldsFromUnixEpochMillis(
+                    CREATE_TIME.getPreferredName(),
+                    CREATE_TIME.getPreferredName() + "_string",
+                    createTime.toEpochMilli()
+                );
             }
             if (version != null) {
                 builder.field(VERSION.getPreferredName(), version);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java
index e61517569445b..61c18c7c84161 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java
@@ -143,7 +143,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(REASON.getPreferredName(), reason);
         }
         if (lastStateChangeTime != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LAST_STATE_CHANGE_TIME.getPreferredName(),
                 LAST_STATE_CHANGE_TIME.getPreferredName() + "_string",
                 lastStateChangeTime.toEpochMilli()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
index de43f744c307b..4e5f97acacf64 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
@@ -145,7 +145,7 @@ default SearchSourceBuilder buildSearch(EvaluationParameters parameters, QueryBu
      */
     default void process(SearchResponse searchResponse) {
         Objects.requireNonNull(searchResponse);
-        if (searchResponse.getHits().getTotalHits().value == 0) {
+        if (searchResponse.getHits().getTotalHits().value() == 0) {
             String requiredFieldsString = String.join(", ", getRequiredFields());
             throw ExceptionsHelper.badRequestException("No documents found containing all the required fields [{}]", requiredFieldsString);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java
index 8b7cff0e80441..0bc191defa6ec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java
@@ -131,7 +131,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
             builder.field(Fields.JOB_ID.getPreferredName(), jobId);
         }
-        builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Fields.TIMESTAMP.getPreferredName(),
+            Fields.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.toEpochMilli()
+        );
         builder.field(ITERATION.getPreferredName(), iteration);
         builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters);
         builder.field(TIMING_STATS.getPreferredName(), timingStats);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java
index 9e9ff3e759e49..c5941cefb1531 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java
@@ -126,7 +126,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.JOB_ID.getPreferredName(), jobId);
         }
         if (timestamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 Fields.TIMESTAMP.getPreferredName(),
                 Fields.TIMESTAMP.getPreferredName() + "_string",
                 timestamp.toEpochMilli()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
index 6ddc078bef4af..b78b495015ab1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
@@ -101,7 +101,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
             builder.field(Fields.JOB_ID.getPreferredName(), jobId);
         }
-        builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Fields.TIMESTAMP.getPreferredName(),
+            Fields.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.toEpochMilli()
+        );
         builder.field(PARAMETERS.getPreferredName(), parameters);
         builder.field(TIMING_STATS.getPreferredName(), timingStats);
         builder.endObject();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java
index 7fff20bcb68ee..c411f3e5353fa 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java
@@ -131,7 +131,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
             builder.field(Fields.JOB_ID.getPreferredName(), jobId);
         }
-        builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Fields.TIMESTAMP.getPreferredName(),
+            Fields.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.toEpochMilli()
+        );
         builder.field(ITERATION.getPreferredName(), iteration);
         builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters);
         builder.field(TIMING_STATS.getPreferredName(), timingStats);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelCacheMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelCacheMetadata.java
index 35c6bf96a09e1..3c8dfecc32e2a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelCacheMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelCacheMetadata.java
@@ -86,7 +86,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_TRAINED_MODEL_CACHE_METADATA_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
index f0909f75d9402..3fb4bb7fa598b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.License;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -509,16 +508,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) {
             builder.field(CREATED_BY.getPreferredName(), createdBy);
             builder.field(VERSION.getPreferredName(), version.toString());
-            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
-            // If we are NOT storing the model, we should return the deprecated field name
-            if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false) == false
-                && builder.getRestApiVersion().matches(RestApiVersion.equalTo(RestApiVersion.V_7))) {
-                builder.humanReadableField(
-                    DEPRECATED_ESTIMATED_HEAP_MEMORY_USAGE_BYTES.getPreferredName(),
-                    ESTIMATED_HEAP_MEMORY_USAGE_HUMAN,
-                    ByteSizeValue.ofBytes(modelSize)
-                );
-            }
+            builder.timestampFieldsFromUnixEpochMillis(
+                CREATE_TIME.getPreferredName(),
+                CREATE_TIME.getPreferredName() + "_string",
+                createTime.toEpochMilli()
+            );
             builder.humanReadableField(MODEL_SIZE_BYTES.getPreferredName(), MODEL_SIZE_HUMAN, ByteSizeValue.ofBytes(modelSize));
             builder.field(ESTIMATED_OPERATIONS.getPreferredName(), estimatedOperations);
             builder.field(LICENSE_LEVEL.getPreferredName(), licenseLevel.description());
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java
index aadaa5254ff15..858d97bf6f956 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java
@@ -297,7 +297,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field("inference_cache_hit_count", cacheHitCount);
             }
             if (lastAccess != null) {
-                builder.timeField("last_access", "last_access_string", lastAccess.toEpochMilli());
+                builder.timestampFieldsFromUnixEpochMillis("last_access", "last_access_string", lastAccess.toEpochMilli());
             }
             if (pendingCount != null) {
                 builder.field("number_of_pending_requests", pendingCount);
@@ -312,7 +312,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field("timeout_count", timeoutCount);
             }
             if (startTime != null) {
-                builder.timeField("start_time", "start_time_string", startTime.toEpochMilli());
+                builder.timestampFieldsFromUnixEpochMillis("start_time", "start_time_string", startTime.toEpochMilli());
             }
             if (threadsPerAllocation != null) {
                 builder.field("threads_per_allocation", threadsPerAllocation);
@@ -608,7 +608,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("cache_size", cacheSize);
         }
         builder.field("priority", priority);
-        builder.timeField("start_time", "start_time_string", startTime.toEpochMilli());
+        builder.timestampFieldsFromUnixEpochMillis("start_time", "start_time_string", startTime.toEpochMilli());
 
         int totalErrorCount = nodeStats.stream().mapToInt(NodeStats::getErrorCount).sum();
         int totalRejectedExecutionCount = nodeStats.stream().mapToInt(NodeStats::getRejectedExecutionCount).sum();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
index 4a87b8e24f481..06c3f75587d62 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
@@ -224,15 +224,12 @@ public boolean hasStartedRoutes() {
         return nodeRoutingTable.values().stream().anyMatch(routeInfo -> routeInfo.getState() == RoutingState.STARTED);
     }
 
-    public List<Tuple<String, Integer>> selectRandomStartedNodesWeighedOnAllocationsForNRequests(
-        int numberOfRequests,
-        RoutingState requiredState
-    ) {
+    public List<Tuple<String, Integer>> selectRandomNodesWeighedOnAllocations(int numberOfRequests, RoutingState... acceptableStates) {
         List<String> nodeIds = new ArrayList<>(nodeRoutingTable.size());
         List<Integer> cumulativeAllocations = new ArrayList<>(nodeRoutingTable.size());
         int allocationSum = 0;
         for (Map.Entry<String, RoutingInfo> routingEntry : nodeRoutingTable.entrySet()) {
-            if (routingEntry.getValue().getState() == requiredState) {
+            if (routingEntry.getValue().getState().isAnyOf(acceptableStates)) {
                 nodeIds.add(routingEntry.getKey());
                 allocationSum += routingEntry.getValue().getCurrentAllocations();
                 cumulativeAllocations.add(allocationSum);
@@ -368,7 +365,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (reason != null) {
             builder.field(REASON.getPreferredName(), reason);
         }
-        builder.timeField(START_TIME.getPreferredName(), startTime);
+        builder.timestampField(START_TIME.getPreferredName(), startTime);
         builder.field(MAX_ASSIGNED_ALLOCATIONS.getPreferredName(), maxAssignedAllocations);
         builder.field(ADAPTIVE_ALLOCATIONS.getPreferredName(), adaptiveAllocationsSettings);
         builder.endObject();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java
index 5314702be0688..1721ae0b21349 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java
@@ -162,7 +162,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(INFERENCE_COUNT.getPreferredName(), inferenceCount);
         builder.field(CACHE_MISS_COUNT.getPreferredName(), cacheMissCount);
         builder.field(MISSING_ALL_FIELDS_COUNT.getPreferredName(), missingAllFieldsCount);
-        builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timeStamp.toEpochMilli());
+        builder.timestampFieldsFromUnixEpochMillis(
+            TIMESTAMP.getPreferredName(),
+            TIMESTAMP.getPreferredName() + "_string",
+            timeStamp.toEpochMilli()
+        );
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java
index d921bc1d4a158..cfbc6c6701427 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java
@@ -260,7 +260,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(MINIMUM_VERSION.getPreferredName(), minimumVersion);
         }
         if (createTime != null) {
-            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
+            builder.timestampFieldsFromUnixEpochMillis(
+                CREATE_TIME.getPreferredName(),
+                CREATE_TIME.getPreferredName() + "_string",
+                createTime.toEpochMilli()
+            );
         }
         if (size > 0) {
             builder.field(SIZE.getPreferredName(), size);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
index 8da0209e10293..e663bbd6800bd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
@@ -613,9 +613,13 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
             if (jobVersion != null) {
                 builder.field(JOB_VERSION.getPreferredName(), jobVersion);
             }
-            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime());
+            builder.timestampFieldsFromUnixEpochMillis(
+                CREATE_TIME.getPreferredName(),
+                CREATE_TIME.getPreferredName() + humanReadableSuffix,
+                createTime.getTime()
+            );
             if (finishedTime != null) {
-                builder.timeField(
+                builder.timestampFieldsFromUnixEpochMillis(
                     FINISHED_TIME.getPreferredName(),
                     FINISHED_TIME.getPreferredName() + humanReadableSuffix,
                     finishedTime.getTime()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java
index 2d03d4273013d..64c449020daa8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java
@@ -150,7 +150,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(REASON.getPreferredName(), reason);
         }
         if (lastStateChangeTime != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LAST_STATE_CHANGE_TIME.getPreferredName(),
                 LAST_STATE_CHANGE_TIME.getPreferredName() + "_string",
                 lastStateChangeTime.toEpochMilli()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
index 6ebed55451ae7..9f9def6a0678d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
@@ -281,6 +281,9 @@ public final class Messages {
     public static final String FIELD_CANNOT_BE_NULL = "Field [{0}] cannot be null";
     public static final String MODEL_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT =
         "Model IDs must be unique. Requested model ID [{}] matches existing model IDs but must not.";
+    public static final String MODEL_ID_DOES_NOT_MATCH_EXISTING_MODEL_IDS_BUT_MUST_FOR_IN_CLUSTER_SERVICE =
+        "Requested model ID [{}] does not have a matching trained model and thus cannot be updated.";
+    public static final String INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE = "The inference endpoint [{}] does not exist and cannot be updated";
 
     private Messages() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java
index 2254959242eab..24a6668a0c016 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java
@@ -99,7 +99,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(ID.getPreferredName(), id);
         if (lastFinalizedBucketEnd != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LAST_FINALIZED_BUCKET_END.getPreferredName(),
                 LAST_FINALIZED_BUCKET_END.getPreferredName() + "_string",
                 lastFinalizedBucketEnd.toEpochMilli()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java
index 91f09bc8171da..fe8cb390db805 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java
@@ -195,8 +195,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(DEAD_CATEGORY_COUNT_FIELD.getPreferredName(), deadCategoryCount);
         builder.field(FAILED_CATEGORY_COUNT_FIELD.getPreferredName(), failedCategoryCount);
         builder.field(CATEGORIZATION_STATUS_FIELD.getPreferredName(), categorizationStatus);
-        builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.toEpochMilli());
-        builder.timeField(TIMESTAMP_FIELD.getPreferredName(), TIMESTAMP_FIELD.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.timestampFieldsFromUnixEpochMillis(
+            LOG_TIME_FIELD.getPreferredName(),
+            LOG_TIME_FIELD.getPreferredName() + "_string",
+            logTime.toEpochMilli()
+        );
+        builder.timestampFieldsFromUnixEpochMillis(
+            TIMESTAMP_FIELD.getPreferredName(),
+            TIMESTAMP_FIELD.getPreferredName() + "_string",
+            timestamp.toEpochMilli()
+        );
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java
index 775640ac2048f..4c9a3a4b70ecb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java
@@ -583,35 +583,35 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         builder.field(SPARSE_BUCKET_COUNT.getPreferredName(), sparseBucketCount);
         builder.field(BUCKET_COUNT.getPreferredName(), bucketCount);
         if (earliestRecordTimeStamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 EARLIEST_RECORD_TIME.getPreferredName(),
                 EARLIEST_RECORD_TIME.getPreferredName() + "_string",
                 earliestRecordTimeStamp.getTime()
             );
         }
         if (latestRecordTimeStamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LATEST_RECORD_TIME.getPreferredName(),
                 LATEST_RECORD_TIME.getPreferredName() + "_string",
                 latestRecordTimeStamp.getTime()
             );
         }
         if (lastDataTimeStamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LAST_DATA_TIME.getPreferredName(),
                 LAST_DATA_TIME.getPreferredName() + "_string",
                 lastDataTimeStamp.getTime()
             );
         }
         if (latestEmptyBucketTimeStamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LATEST_EMPTY_BUCKET_TIME.getPreferredName(),
                 LATEST_EMPTY_BUCKET_TIME.getPreferredName() + "_string",
                 latestEmptyBucketTimeStamp.getTime()
             );
         }
         if (latestSparseBucketTimeStamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LATEST_SPARSE_BUCKET_TIME.getPreferredName(),
                 LATEST_SPARSE_BUCKET_TIME.getPreferredName() + "_string",
                 latestSparseBucketTimeStamp.getTime()
@@ -619,7 +619,11 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         }
         builder.field(INPUT_RECORD_COUNT.getPreferredName(), getInputRecordCount());
         if (logTime != null) {
-            builder.timeField(LOG_TIME.getPreferredName(), LOG_TIME.getPreferredName() + "_string", logTime.toEpochMilli());
+            builder.timestampFieldsFromUnixEpochMillis(
+                LOG_TIME.getPreferredName(),
+                LOG_TIME.getPreferredName() + "_string",
+                logTime.toEpochMilli()
+            );
         }
 
         return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java
index 16eceb1e89a95..3a9123445e697 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java
@@ -264,7 +264,7 @@ public ModelSizeStats(StreamInput in) throws IOException {
         } else {
             assignmentMemoryBasis = null;
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             outputMemoryAllocatorBytes = in.readOptionalVLong();
         } else {
             outputMemoryAllocatorBytes = null;
@@ -306,7 +306,7 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(false);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeOptionalVLong(outputMemoryAllocatorBytes);
         }
         out.writeVLong(categorizedDocCount);
@@ -363,9 +363,17 @@ public XContentBuilder doXContentBody(XContentBuilder builder) throws IOExceptio
         builder.field(DEAD_CATEGORY_COUNT_FIELD.getPreferredName(), deadCategoryCount);
         builder.field(FAILED_CATEGORY_COUNT_FIELD.getPreferredName(), failedCategoryCount);
         builder.field(CATEGORIZATION_STATUS_FIELD.getPreferredName(), categorizationStatus);
-        builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            LOG_TIME_FIELD.getPreferredName(),
+            LOG_TIME_FIELD.getPreferredName() + "_string",
+            logTime.getTime()
+        );
         if (timestamp != null) {
-            builder.timeField(TIMESTAMP_FIELD.getPreferredName(), TIMESTAMP_FIELD.getPreferredName() + "_string", timestamp.getTime());
+            builder.timestampFieldsFromUnixEpochMillis(
+                TIMESTAMP_FIELD.getPreferredName(),
+                TIMESTAMP_FIELD.getPreferredName() + "_string",
+                timestamp.getTime()
+            );
         }
 
         return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
index bf62a8a267f84..3114c03879eb7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
@@ -194,7 +194,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Job.ID.getPreferredName(), jobId);
         builder.field(MIN_VERSION.getPreferredName(), minVersion);
         if (timestamp != null) {
-            builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+            builder.timestampFieldsFromUnixEpochMillis(
+                TIMESTAMP.getPreferredName(),
+                TIMESTAMP.getPreferredName() + "_string",
+                timestamp.getTime()
+            );
         }
         if (description != null) {
             builder.field(DESCRIPTION.getPreferredName(), description);
@@ -207,14 +211,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName(), modelSizeStats);
         }
         if (latestRecordTimeStamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LATEST_RECORD_TIME.getPreferredName(),
                 LATEST_RECORD_TIME.getPreferredName() + "_string",
                 latestRecordTimeStamp.getTime()
             );
         }
         if (latestResultTimeStamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LATEST_RESULT_TIME.getPreferredName(),
                 LATEST_RESULT_TIME.getPreferredName() + "_string",
                 latestResultTimeStamp.getTime()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
index ca1fd98b7bfb3..3b4d5b6a72654 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
@@ -283,7 +283,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
         builder.field(Detector.DETECTOR_INDEX.getPreferredName(), detectorIndex);
         builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
-        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Result.TIMESTAMP.getPreferredName(),
+            Result.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.getTime()
+        );
         if (byFieldName != null) {
             builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java
index b4798b404a434..f867511d992c6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java
@@ -173,7 +173,11 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(JOB_ID.getPreferredName(), jobId);
-        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Result.TIMESTAMP.getPreferredName(),
+            Result.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.getTime()
+        );
         builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore);
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
         builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java
index f659ceced3565..131e0c24b387e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java
@@ -132,7 +132,11 @@ XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws I
         builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore);
         builder.field(RAW_ANOMALY_SCORE.getPreferredName(), rawAnomalyScore);
         builder.field(PROBABILITY.getPreferredName(), probability);
-        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Result.TIMESTAMP.getPreferredName(),
+            Result.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.getTime()
+        );
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
         builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
         return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java
index 20a2fa95b08f3..37eba1fc081a0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java
@@ -140,7 +140,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
         builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex);
         if (timestamp != null) {
-            builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+            builder.timestampFieldsFromUnixEpochMillis(
+                Result.TIMESTAMP.getPreferredName(),
+                Result.TIMESTAMP.getPreferredName() + "_string",
+                timestamp.getTime()
+            );
         }
         if (partitionFieldName != null) {
             builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java
index b544c43295bc5..930c8b6f3ef68 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java
@@ -132,7 +132,11 @@ XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws I
         builder.field(PROBABILITY.getPreferredName(), probability);
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
         builder.field(Result.IS_INTERIM.getPreferredName(), isInterim);
-        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Result.TIMESTAMP.getPreferredName(),
+            Result.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.getTime()
+        );
         return builder;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java
index ba1a03c64e15e..043611f3333f6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java
@@ -153,7 +153,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex);
 
         if (timestamp != null) {
-            builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+            builder.timestampFieldsFromUnixEpochMillis(
+                Result.TIMESTAMP.getPreferredName(),
+                Result.TIMESTAMP.getPreferredName() + "_string",
+                timestamp.getTime()
+            );
         }
         if (partitionFieldName != null) {
             builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java
index c04a61951ad99..8cdcaa0205b0f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java
@@ -71,7 +71,11 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
+        builder.timestampFieldsFromUnixEpochMillis(
+            Result.TIMESTAMP.getPreferredName(),
+            Result.TIMESTAMP.getPreferredName() + "_string",
+            timestamp.getTime()
+        );
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
         builder.field(OVERALL_SCORE.getPreferredName(), overallScore);
         builder.field(JOBS.getPreferredName(), jobs);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
index fb75d95aeed1b..73e3c31297fbf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
@@ -93,6 +93,10 @@ public static ElasticsearchStatusException badRequestException(String msg, Objec
         return new ElasticsearchStatusException(msg, RestStatus.BAD_REQUEST, args);
     }
 
+    public static ElasticsearchStatusException entityNotFoundException(String msg, Object... args) {
+        return new ElasticsearchStatusException(msg, RestStatus.NOT_FOUND, args);
+    }
+
     public static ElasticsearchStatusException taskOperationFailureToStatusException(TaskOperationFailure failure) {
         return new ElasticsearchStatusException(failure.getCause().getMessage(), failure.getStatus(), failure.getCause());
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java
index 39d822b843d15..e102f0712b283 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java
@@ -178,7 +178,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(INCREMENTAL_METRIC_VALUE_MS.getPreferredName(), incrementalMetricValueMs);
         if (latestTimestamp != null) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 LATEST_TIMESTAMP.getPreferredName(),
                 LATEST_TIMESTAMP.getPreferredName() + "_string",
                 latestTimestamp.toEpochMilli()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java
index e61342d281c90..83f7832645270 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java
@@ -10,9 +10,7 @@
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestMetadata;
-import org.elasticsearch.ingest.Pipeline;
 import org.elasticsearch.transport.Transports;
 
 import java.util.HashMap;
@@ -24,6 +22,7 @@
 import java.util.function.Consumer;
 
 import static org.elasticsearch.inference.InferenceResults.MODEL_ID_RESULTS_FIELD;
+import static org.elasticsearch.ingest.Pipeline.ON_FAILURE_KEY;
 import static org.elasticsearch.ingest.Pipeline.PROCESSORS_KEY;
 
 /**
@@ -53,16 +52,10 @@ public static int countInferenceProcessors(ClusterState state) {
         Counter counter = Counter.newCounter();
         ingestMetadata.getPipelines().forEach((pipelineId, configuration) -> {
             Map<String, Object> configMap = configuration.getConfigAsMap();
-            List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, configMap, PROCESSORS_KEY);
+            List<Map<String, Object>> processorConfigs = (List<Map<String, Object>>) configMap.get(PROCESSORS_KEY);
             for (Map<String, Object> processorConfigWithKey : processorConfigs) {
                 for (Map.Entry<String, Object> entry : processorConfigWithKey.entrySet()) {
-                    addModelsAndPipelines(
-                        entry.getKey(),
-                        pipelineId,
-                        (Map<String, Object>) entry.getValue(),
-                        pam -> counter.addAndGet(1),
-                        0
-                    );
+                    addModelsAndPipelines(entry.getKey(), pipelineId, entry.getValue(), pam -> counter.addAndGet(1), 0);
                 }
             }
         });
@@ -73,7 +66,6 @@ public static int countInferenceProcessors(ClusterState state) {
      * @param ingestMetadata The ingestMetadata of current ClusterState
      * @return The set of model IDs referenced by inference processors
      */
-    @SuppressWarnings("unchecked")
     public static Set<String> getModelIdsFromInferenceProcessors(IngestMetadata ingestMetadata) {
         if (ingestMetadata == null) {
             return Set.of();
@@ -82,7 +74,7 @@ public static Set<String> getModelIdsFromInferenceProcessors(IngestMetadata inge
         Set<String> modelIds = new LinkedHashSet<>();
         ingestMetadata.getPipelines().forEach((pipelineId, configuration) -> {
             Map<String, Object> configMap = configuration.getConfigAsMap();
-            List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, configMap, PROCESSORS_KEY);
+            List<Map<String, Object>> processorConfigs = readList(configMap, PROCESSORS_KEY);
             for (Map<String, Object> processorConfigWithKey : processorConfigs) {
                 for (Map.Entry<String, Object> entry : processorConfigWithKey.entrySet()) {
                     addModelsAndPipelines(entry.getKey(), pipelineId, entry.getValue(), pam -> modelIds.add(pam.modelIdOrAlias()), 0);
@@ -96,7 +88,6 @@ public static Set<String> getModelIdsFromInferenceProcessors(IngestMetadata inge
      * @param state Current cluster state
      * @return a map from Model or Deployment IDs or Aliases to each pipeline referencing them.
      */
-    @SuppressWarnings("unchecked")
     public static Map<String, Set<String>> pipelineIdsByResource(ClusterState state, Set<String> ids) {
         assert Transports.assertNotTransportThread("non-trivial nested loops over cluster state structures");
         Map<String, Set<String>> pipelineIdsByModelIds = new HashMap<>();
@@ -110,7 +101,7 @@ public static Map<String, Set<String>> pipelineIdsByResource(ClusterState state,
         }
         ingestMetadata.getPipelines().forEach((pipelineId, configuration) -> {
             Map<String, Object> configMap = configuration.getConfigAsMap();
-            List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, configMap, PROCESSORS_KEY);
+            List<Map<String, Object>> processorConfigs = readList(configMap, PROCESSORS_KEY);
             for (Map<String, Object> processorConfigWithKey : processorConfigs) {
                 for (Map.Entry<String, Object> entry : processorConfigWithKey.entrySet()) {
                     addModelsAndPipelines(entry.getKey(), pipelineId, entry.getValue(), pam -> {
@@ -128,7 +119,6 @@ public static Map<String, Set<String>> pipelineIdsByResource(ClusterState state,
      * @param state Current {@link ClusterState}
      * @return a map from Model or Deployment IDs or Aliases to each pipeline referencing them.
      */
-    @SuppressWarnings("unchecked")
     public static Set<String> pipelineIdsForResource(ClusterState state, Set<String> ids) {
         assert Transports.assertNotTransportThread("non-trivial nested loops over cluster state structures");
         Set<String> pipelineIds = new HashSet<>();
@@ -142,7 +132,7 @@ public static Set<String> pipelineIdsForResource(ClusterState state, Set<String>
         }
         ingestMetadata.getPipelines().forEach((pipelineId, configuration) -> {
             Map<String, Object> configMap = configuration.getConfigAsMap();
-            List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, configMap, PROCESSORS_KEY);
+            List<Map<String, Object>> processorConfigs = readList(configMap, PROCESSORS_KEY);
             for (Map<String, Object> processorConfigWithKey : processorConfigs) {
                 for (Map.Entry<String, Object> entry : processorConfigWithKey.entrySet()) {
                     addModelsAndPipelines(entry.getKey(), pipelineId, entry.getValue(), pam -> {
@@ -188,7 +178,7 @@ private static void addModelsAndPipelines(
                     addModelsAndPipelines(
                         innerProcessorWithName.getKey(),
                         pipelineId,
-                        (Map<String, Object>) innerProcessorWithName.getValue(),
+                        innerProcessorWithName.getValue(),
                         handler,
                         level + 1
                     );
@@ -196,13 +186,8 @@ private static void addModelsAndPipelines(
             }
             return;
         }
-        if (processorDefinition instanceof Map<?, ?> definitionMap && definitionMap.containsKey(Pipeline.ON_FAILURE_KEY)) {
-            List<Map<String, Object>> onFailureConfigs = ConfigurationUtils.readList(
-                null,
-                null,
-                (Map<String, Object>) definitionMap,
-                Pipeline.ON_FAILURE_KEY
-            );
+        if (processorDefinition instanceof Map<?, ?> definitionMap && definitionMap.containsKey(ON_FAILURE_KEY)) {
+            List<Map<String, Object>> onFailureConfigs = readList(definitionMap, ON_FAILURE_KEY);
             onFailureConfigs.stream()
                 .flatMap(map -> map.entrySet().stream())
                 .forEach(entry -> addModelsAndPipelines(entry.getKey(), pipelineId, entry.getValue(), handler, level + 1));
@@ -211,4 +196,16 @@ private static void addModelsAndPipelines(
 
     private record PipelineAndModel(String pipelineId, String modelIdOrAlias) {}
 
+    /**
+     * A local alternative to ConfigurationUtils.readList(...) that reads list properties out of the processor configuration map,
+     * but doesn't rely on mutating the configuration map.
+     */
+    @SuppressWarnings("unchecked")
+    private static List<Map<String, Object>> readList(Map<?, ?> processorConfig, String key) {
+        Object val = processorConfig.get(key);
+        if (val == null) {
+            throw new IllegalArgumentException("Missing required property [" + key + "]");
+        }
+        return (List<Map<String, Object>>) val;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java
index cfa2efc53ce96..4e558e2a394dc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -22,7 +22,6 @@
 import java.util.EnumSet;
 import java.util.List;
 
-import static org.elasticsearch.core.RestApiVersion.onOrAfter;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 
@@ -42,6 +41,7 @@ public String getName() {
     }
 
     @Override
+    @UpdateForV9(owner = UpdateForV9.Owner.SECURITY) // accept_enterprise parameter no longer supported?
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
 
         // we piggyback verbosity on "human" output
@@ -56,8 +56,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
                 "Including [accept_enterprise] in get license requests is deprecated."
                     + " The parameter will be removed in the next major version"
             );
-            if (request.paramAsBoolean("accept_enterprise", true) == false
-                && request.getRestApiVersion().matches(onOrAfter(RestApiVersion.V_8))) {
+            if (request.paramAsBoolean("accept_enterprise", true) == false) {
                 throw new IllegalArgumentException("The [accept_enterprise] parameters may not be false");
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java
index a198c0570cd91..6b4ee3c1fdd16 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java
@@ -16,15 +16,13 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.TransportVersions.ROLLUP_USAGE;
-
 public class RollupFeatureSetUsage extends XPackFeatureSet.Usage {
 
     private final int numberOfRollupJobs;
 
     public RollupFeatureSetUsage(StreamInput input) throws IOException {
         super(input);
-        this.numberOfRollupJobs = input.getTransportVersion().onOrAfter(ROLLUP_USAGE) ? input.readVInt() : 0;
+        this.numberOfRollupJobs = input.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? input.readVInt() : 0;
     }
 
     public RollupFeatureSetUsage(int numberOfRollupJobs) {
@@ -39,7 +37,7 @@ public int getNumberOfRollupJobs() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getTransportVersion().onOrAfter(ROLLUP_USAGE)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeVInt(numberOfRollupJobs);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
index b632c680260cf..32b401ebfb32d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
@@ -238,12 +238,16 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
             }
             builder.field("is_partial", isPartial);
             builder.field("is_running", isRunning);
-            builder.timeField("start_time_in_millis", "start_time", startTimeMillis);
-            builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
+            builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTimeMillis);
+            builder.timestampFieldsFromUnixEpochMillis("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
             if (searchResponse != null) {
                 if (isRunning == false) {
                     TimeValue took = searchResponse.getTook();
-                    builder.timeField("completion_time_in_millis", "completion_time", startTimeMillis + took.millis());
+                    builder.timestampFieldsFromUnixEpochMillis(
+                        "completion_time_in_millis",
+                        "completion_time",
+                        startTimeMillis + took.millis()
+                    );
                 }
                 builder.field("response");
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java
index 10b7730b58c9b..89d4be514adde 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java
@@ -175,10 +175,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("id", id);
         builder.field("is_running", isRunning);
         builder.field("is_partial", isPartial);
-        builder.timeField("start_time_in_millis", "start_time", startTimeMillis);
-        builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
+        builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTimeMillis);
+        builder.timestampFieldsFromUnixEpochMillis("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
         if (completionTimeMillis != null) {
-            builder.timeField("completion_time_in_millis", "completion_time", completionTimeMillis);
+            builder.timestampFieldsFromUnixEpochMillis("completion_time_in_millis", "completion_time", completionTimeMillis);
         }
         RestActions.buildBroadcastShardsHeader(builder, params, totalShards, successfulShards, skippedShards, failedShards, null);
         if (clusters != null) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java
index 3154fe5999b8e..129619f6976e2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java
@@ -79,19 +79,19 @@ public void onResponse(SearchResponse resp) {
                                     }
                                 }
 
-                                if (results.size() > resp.getHits().getTotalHits().value) {
+                                if (results.size() > resp.getHits().getTotalHits().value()) {
                                     clearScroll.accept(lastResponse);
                                     listener.onFailure(
                                         new IllegalStateException(
                                             "scrolling returned more hits ["
                                                 + results.size()
                                                 + "] than expected ["
-                                                + resp.getHits().getTotalHits().value
+                                                + resp.getHits().getTotalHits().value()
                                                 + "] so bailing out to prevent unbounded "
                                                 + "memory consumption."
                                         )
                                     );
-                                } else if (results.size() == resp.getHits().getTotalHits().value) {
+                                } else if (results.size() == resp.getHits().getTotalHits().value()) {
                                     clearScroll.accept(resp);
                                     // Finally, return the list of the entity
                                     listener.onResponse(Collections.unmodifiableList(results));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/profile/SuggestProfilesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/profile/SuggestProfilesResponse.java
index 0061870c73cc9..32b12c834dd9c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/profile/SuggestProfilesResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/profile/SuggestProfilesResponse.java
@@ -55,8 +55,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("took", tookInMillis);
             builder.startObject("total");
             {
-                builder.field("value", totalHits.value);
-                builder.field("relation", totalHits.relation == TotalHits.Relation.EQUAL_TO ? "eq" : "gte");
+                builder.field("value", totalHits.value());
+                builder.field("relation", totalHits.relation() == TotalHits.Relation.EQUAL_TO ? "eq" : "gte");
             }
             builder.endObject();
             builder.startArray("profiles");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/GetSecuritySettingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/GetSecuritySettingsAction.java
index 093dfabd0bf65..88bc8a46532b5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/GetSecuritySettingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/GetSecuritySettingsAction.java
@@ -42,7 +42,7 @@ public Request(TimeValue masterNodeTimeout) {
 
         @UpdateForV9(owner = UpdateForV9.Owner.SECURITY) // no need for bwc any more, this can be inlined
         public static Request readFrom(StreamInput in) throws IOException {
-            if (in.getTransportVersion().onOrAfter(TransportVersions.SECURITY_SETTINGS_REQUEST_TIMEOUTS)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 return new Request(in);
             } else {
                 return new Request(TimeValue.THIRTY_SECONDS);
@@ -55,7 +55,7 @@ private Request(StreamInput in) throws IOException {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().onOrAfter(TransportVersions.SECURITY_SETTINGS_REQUEST_TIMEOUTS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 super.writeTo(out);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java
index 316f162729356..c3bd1c51d113c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java
@@ -118,7 +118,7 @@ public Request(
 
         @UpdateForV9(owner = UpdateForV9.Owner.SECURITY) // no need for bwc any more, this can be inlined
         public static Request readFrom(StreamInput in) throws IOException {
-            if (in.getTransportVersion().onOrAfter(TransportVersions.SECURITY_SETTINGS_REQUEST_TIMEOUTS)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 return new Request(in);
             } else {
                 return new Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, in);
@@ -141,7 +141,7 @@ private Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, StreamInput i
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().onOrAfter(TransportVersions.SECURITY_SETTINGS_REQUEST_TIMEOUTS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 super.writeTo(out);
             }
             out.writeGenericMap(this.mainIndexSettings);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
index c5cbe50ef1575..de351cd59c690 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
@@ -30,6 +30,8 @@
 import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
+
 /**
  * Response for a {@link GetUserPrivilegesRequest}
  */
@@ -55,7 +57,7 @@ public GetUserPrivilegesResponse(StreamInput in) throws IOException {
         } else {
             remoteIndex = Set.of();
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS)) {
+        if (in.getTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS)) {
             remoteClusterPermissions = new RemoteClusterPermissions(in);
         } else {
             remoteClusterPermissions = RemoteClusterPermissions.NONE;
@@ -134,14 +136,14 @@ public void writeTo(StreamOutput out) throws IOException {
                     + "]"
             );
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS)) {
+        if (out.getTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS)) {
             remoteClusterPermissions.writeTo(out);
         } else if (hasRemoteClusterPrivileges()) {
             throw new IllegalArgumentException(
                 "versions of Elasticsearch before ["
-                    + TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS
+                    + ROLE_REMOTE_CLUSTER_PRIVS.toReleaseVersion()
                     + "] can't handle remote cluster privileges and attempted to send to ["
-                    + out.getTransportVersion()
+                    + out.getTransportVersion().toReleaseVersion()
                     + "]"
             );
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index 3e3b1178b372c..04dda75692208 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -54,7 +54,6 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -77,6 +76,7 @@
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.FALLBACK_REALM_NAME;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.FALLBACK_REALM_TYPE;
 import static org.elasticsearch.xpack.core.security.authc.RealmDomain.REALM_DOMAIN_PARSER;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 
 /**
  * The Authentication class encapsulates identity information created after successful authentication
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
index 17088cff8718b..c504ebe56ed45 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
@@ -54,6 +54,18 @@
  */
 public class ExpressionRoleMapping implements ToXContentObject, Writeable {
 
+    /**
+     * Reserved suffix for read-only operator-defined role mappings.
+     * This suffix is added to the name of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_SUFFIX = "-read-only-operator-mapping";
+    /**
+     * Reserved metadata field to mark role mappings as read-only.
+     * This field is added to the metadata of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_METADATA_FLAG = "_read_only";
     private static final ObjectParser<Builder, String> PARSER = new ObjectParser<>("role-mapping", Builder::new);
 
     /**
@@ -136,6 +148,28 @@ public ExpressionRoleMapping(StreamInput in) throws IOException {
         this.metadata = in.readGenericMap();
     }
 
+    public static boolean hasReadOnlySuffix(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX);
+    }
+
+    public static void validateNoReadOnlySuffix(String name) {
+        if (hasReadOnlySuffix(name)) {
+            throw new IllegalArgumentException(
+                "Invalid mapping name [" + name + "]. [" + READ_ONLY_ROLE_MAPPING_SUFFIX + "] is not an allowed suffix"
+            );
+        }
+    }
+
+    public static String addReadOnlySuffix(String name) {
+        return name + READ_ONLY_ROLE_MAPPING_SUFFIX;
+    }
+
+    public static String removeReadOnlySuffixIfPresent(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX)
+            ? name.substring(0, name.length() - READ_ONLY_ROLE_MAPPING_SUFFIX.length())
+            : name;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
index 1a8839fa0fa4a..8d069caf0496f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
@@ -50,6 +50,7 @@
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.createParserNotCompressed;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 
 /**
  * A holder for a Role that contains user-readable information about the Role
@@ -58,6 +59,7 @@
 public class RoleDescriptor implements ToXContentObject, Writeable {
 
     public static final TransportVersion WORKFLOWS_RESTRICTION_VERSION = TransportVersions.V_8_9_X;
+    public static final TransportVersion SECURITY_ROLE_DESCRIPTION = TransportVersions.V_8_15_0;
 
     public static final String ROLE_TYPE = "role";
 
@@ -220,12 +222,12 @@ public RoleDescriptor(StreamInput in) throws IOException {
         } else {
             this.restriction = Restriction.NONE;
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS)) {
+        if (in.getTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS)) {
             this.remoteClusterPermissions = new RemoteClusterPermissions(in);
         } else {
             this.remoteClusterPermissions = RemoteClusterPermissions.NONE;
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.SECURITY_ROLE_DESCRIPTION)) {
+        if (in.getTransportVersion().onOrAfter(SECURITY_ROLE_DESCRIPTION)) {
             this.description = in.readOptionalString();
         } else {
             this.description = "";
@@ -485,10 +487,10 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(WORKFLOWS_RESTRICTION_VERSION)) {
             restriction.writeTo(out);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS)) {
+        if (out.getTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS)) {
             remoteClusterPermissions.writeTo(out);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.SECURITY_ROLE_DESCRIPTION)) {
+        if (out.getTransportVersion().onOrAfter(SECURITY_ROLE_DESCRIPTION)) {
             out.writeOptionalString(description);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
index da6ff6ad24c34..74c6223b1ebdd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.core.security.authz;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
@@ -26,8 +28,10 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -36,7 +40,11 @@
 
 public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
 
+    private static final Logger logger = LogManager.getLogger(RoleMappingMetadata.class);
+
     public static final String TYPE = "role_mappings";
+    public static final String METADATA_NAME_FIELD = "_es_reserved_role_mapping_name";
+    public static final String FALLBACK_NAME = "name_not_available_after_deserialization";
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<RoleMappingMetadata, Void> PARSER = new ConstructingObjectParser<>(
@@ -46,12 +54,7 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Cu
     );
 
     static {
-        PARSER.declareObjectArray(
-            constructorArg(),
-            // role mapping names are lost when the role mapping metadata is serialized
-            (p, c) -> ExpressionRoleMapping.parse("name_not_available_after_deserialization", p),
-            new ParseField(TYPE)
-        );
+        PARSER.declareObjectArray(constructorArg(), (p, c) -> parseWithNameFromMetadata(p), new ParseField(TYPE));
     }
 
     private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of());
@@ -108,7 +111,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.SECURITY_ROLE_MAPPINGS_IN_CLUSTER_STATE;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
@@ -153,4 +156,64 @@ public EnumSet<Metadata.XContentContext> context() {
         // are not persisted.
         return ALL_CONTEXTS;
     }
+
+    /**
+     * Ensures role mapping names are preserved when stored on disk using XContent format,
+     * which omits names. This method copies the role mapping's name into a reserved metadata field
+     * during serialization, allowing recovery during deserialization (e.g., after a master-node restart).
+     * {@link #parseWithNameFromMetadata(XContentParser)} restores the name during parsing.
+     */
+    public static ExpressionRoleMapping copyWithNameInMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = new HashMap<>(roleMapping.getMetadata());
+        // note: can't use Maps.copyWith... since these create maps that don't support `null` values in map entries
+        if (metadata.put(METADATA_NAME_FIELD, roleMapping.getName()) != null) {
+            logger.error(
+                "Metadata field [{}] is reserved and will be overwritten with an internal system value. "
+                    + "Rename this field in your role mapping configuration.",
+                METADATA_NAME_FIELD
+            );
+        }
+        return new ExpressionRoleMapping(
+            roleMapping.getName(),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            metadata,
+            roleMapping.isEnabled()
+        );
+    }
+
+    /**
+     * If a role mapping does not yet have a name persisted in metadata, it will use a constant fallback name. This method checks if a
+     * role mapping has the fallback name.
+     */
+    public static boolean hasFallbackName(ExpressionRoleMapping expressionRoleMapping) {
+        return expressionRoleMapping.getName().equals(FALLBACK_NAME);
+    }
+
+    /**
+     * Parse a role mapping from XContent, restoring the name from a reserved metadata field.
+     * Used to parse a role mapping annotated with its name in metadata via @see {@link #copyWithNameInMetadata(ExpressionRoleMapping)}.
+     */
+    public static ExpressionRoleMapping parseWithNameFromMetadata(XContentParser parser) throws IOException {
+        ExpressionRoleMapping roleMapping = ExpressionRoleMapping.parse(FALLBACK_NAME, parser);
+        return new ExpressionRoleMapping(
+            getNameFromMetadata(roleMapping),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            roleMapping.getMetadata(),
+            roleMapping.isEnabled()
+        );
+    }
+
+    private static String getNameFromMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = roleMapping.getMetadata();
+        if (metadata.containsKey(METADATA_NAME_FIELD) && metadata.get(METADATA_NAME_FIELD) instanceof String name) {
+            return name;
+        } else {
+            // This is valid the first time we recover from cluster-state: the old format metadata won't have a name stored in metadata yet
+            return FALLBACK_NAME;
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
index 09a49c53ee1a5..908f58c5f9147 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
@@ -155,17 +155,6 @@ public FieldInfos getFieldInfos() {
         return fieldInfos;
     }
 
-    @Override
-    public Fields getTermVectors(int docID) throws IOException {
-        Fields f = super.getTermVectors(docID);
-        if (f == null) {
-            return null;
-        }
-        f = new FieldFilterFields(f);
-        // we need to check for emptyness, so we can return null:
-        return f.iterator().hasNext() ? f : null;
-    }
-
     @Override
     public TermVectors termVectors() throws IOException {
         TermVectors termVectors = super.termVectors();
@@ -264,11 +253,6 @@ private static int step(CharacterRunAutomaton automaton, String key, int state)
         return state;
     }
 
-    @Override
-    public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {
-        super.document(docID, new FieldSubsetStoredFieldVisitor(visitor));
-    }
-
     @Override
     protected StoredFieldsReader doGetSequentialStoredFieldsReader(StoredFieldsReader reader) {
         return new FieldSubsetStoredFieldsReader(reader);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
index c85a648761ca7..23c93226d5494 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
@@ -53,7 +53,7 @@ public final class ApplicationPermission {
                 return new PermissionEntry(
                     appPriv,
                     Sets.union(existing.resourceNames, resourceNames),
-                    Automatons.unionAndMinimize(Arrays.asList(existing.resourceAutomaton, patterns))
+                    Automatons.unionAndDeterminize(Arrays.asList(existing.resourceAutomaton, patterns))
                 );
             }
         }));
@@ -187,7 +187,7 @@ private PermissionEntry(ApplicationPrivilege privilege, Set<String> resourceName
         }
 
         private boolean grants(ApplicationPrivilege other, Automaton resource) {
-            return matchesPrivilege(other) && Operations.subsetOf(resource, this.resourceAutomaton);
+            return matchesPrivilege(other) && Automatons.subsetOf(resource, this.resourceAutomaton);
         }
 
         private boolean matchesPrivilege(ApplicationPrivilege other) {
@@ -202,7 +202,7 @@ private boolean matchesPrivilege(ApplicationPrivilege other) {
             }
             return Operations.isEmpty(privilege.getAutomaton()) == false
                 && Operations.isEmpty(other.getAutomaton()) == false
-                && Operations.subsetOf(other.getAutomaton(), privilege.getAutomaton());
+                && Automatons.subsetOf(other.getAutomaton(), privilege.getAutomaton());
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
index 9c41786f39eeb..5f3da8f73a708 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.core.security.authz.permission;
 
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
@@ -138,7 +137,7 @@ public ClusterPermission build() {
             }
             List<PermissionCheck> checks = this.permissionChecks;
             if (false == actionAutomatons.isEmpty()) {
-                final Automaton mergedAutomaton = Automatons.unionAndMinimize(this.actionAutomatons);
+                final Automaton mergedAutomaton = Automatons.unionAndDeterminize(this.actionAutomatons);
                 checks = new ArrayList<>(this.permissionChecks.size() + 1);
                 checks.add(new AutomatonPermissionCheck(mergedAutomaton));
                 checks.addAll(this.permissionChecks);
@@ -157,7 +156,7 @@ private static Automaton createAutomaton(Set<String> allowedActionPatterns, Set<
             } else {
                 final Automaton allowedAutomaton = Automatons.patterns(allowedActionPatterns);
                 final Automaton excludedAutomaton = Automatons.patterns(excludeActionPatterns);
-                return Automatons.minusAndMinimize(allowedAutomaton, excludedAutomaton);
+                return Automatons.minusAndDeterminize(allowedAutomaton, excludedAutomaton);
             }
         }
     }
@@ -215,7 +214,7 @@ public final boolean check(final String action, final TransportRequest request,
         @Override
         public final boolean implies(final PermissionCheck permissionCheck) {
             if (permissionCheck instanceof ActionBasedPermissionCheck) {
-                return Operations.subsetOf(((ActionBasedPermissionCheck) permissionCheck).automaton, this.automaton)
+                return Automatons.subsetOf(((ActionBasedPermissionCheck) permissionCheck).automaton, this.automaton)
                     && doImplies((ActionBasedPermissionCheck) permissionCheck);
             }
             return false;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java
index f3c2d9f62e40f..ed7bbf9158278 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.Strings;
@@ -34,8 +33,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.apache.lucene.util.automaton.Operations.subsetOf;
-
 /**
  * Stores patterns to fields which access is granted or denied to and maintains an automaton that can be used to check if permission is
  * allowed for a specific field.
@@ -150,7 +147,7 @@ public static Automaton initializePermittedFieldsAutomaton(FieldPermissionsDefin
         List<Automaton> automatonList = groups.stream()
             .map(g -> FieldPermissions.buildPermittedFieldsAutomaton(g.getGrantedFields(), g.getExcludedFields()))
             .collect(Collectors.toList());
-        return Automatons.unionAndMinimize(automatonList);
+        return Automatons.unionAndDeterminize(automatonList);
     }
 
     /**
@@ -175,10 +172,14 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel
             deniedFieldsAutomaton = Automatons.patterns(deniedFields);
         }
 
-        grantedFieldsAutomaton = MinimizationOperations.minimize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
-        deniedFieldsAutomaton = MinimizationOperations.minimize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        grantedFieldsAutomaton = Operations.removeDeadStates(
+            Operations.determinize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
+        );
+        deniedFieldsAutomaton = Operations.removeDeadStates(
+            Operations.determinize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
+        );
 
-        if (subsetOf(deniedFieldsAutomaton, grantedFieldsAutomaton) == false) {
+        if (Automatons.subsetOf(deniedFieldsAutomaton, grantedFieldsAutomaton) == false) {
             throw new ElasticsearchSecurityException(
                 "Exceptions for field permissions must be a subset of the "
                     + "granted fields but "
@@ -188,7 +189,7 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel
             );
         }
 
-        grantedFieldsAutomaton = Automatons.minusAndMinimize(grantedFieldsAutomaton, deniedFieldsAutomaton);
+        grantedFieldsAutomaton = Automatons.minusAndDeterminize(grantedFieldsAutomaton, deniedFieldsAutomaton);
         return grantedFieldsAutomaton;
     }
 
@@ -205,7 +206,10 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel
     public FieldPermissions limitFieldPermissions(FieldPermissions limitedBy) {
         if (hasFieldLevelSecurity() && limitedBy != null && limitedBy.hasFieldLevelSecurity()) {
             // TODO: cache the automaton computation with FieldPermissionsCache
-            Automaton _permittedFieldsAutomaton = Automatons.intersectAndMinimize(getIncludeAutomaton(), limitedBy.getIncludeAutomaton());
+            Automaton _permittedFieldsAutomaton = Automatons.intersectAndDeterminize(
+                getIncludeAutomaton(),
+                limitedBy.getIncludeAutomaton()
+            );
             return new FieldPermissions(
                 CollectionUtils.concatLists(fieldPermissionsDefinitions, limitedBy.fieldPermissionsDefinitions),
                 _permittedFieldsAutomaton
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java
index 46261937a0228..a1e14bfde8aa5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java
@@ -107,7 +107,7 @@ FieldPermissions union(Collection<FieldPermissions> fieldPermissionsCollection)
                     List<Automaton> automatonList = fieldPermissionsCollection.stream()
                         .map(FieldPermissions::getIncludeAutomaton)
                         .collect(Collectors.toList());
-                    return new FieldPermissions(key, Automatons.unionAndMinimize(automatonList));
+                    return new FieldPermissions(key, Automatons.unionAndDeterminize(automatonList));
                 });
             } catch (ExecutionException e) {
                 throw new ElasticsearchException("unable to compute field permissions", e);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
index e1b72cc43b38e..cdd5a6f6ff72d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
@@ -283,14 +283,14 @@ public boolean checkResourcePrivileges(
         for (String forIndexPattern : checkForIndexPatterns) {
             Automaton checkIndexAutomaton = Automatons.patterns(forIndexPattern);
             if (false == allowRestrictedIndices && false == isConcreteRestrictedIndex(forIndexPattern)) {
-                checkIndexAutomaton = Automatons.minusAndMinimize(checkIndexAutomaton, restrictedIndices.getAutomaton());
+                checkIndexAutomaton = Automatons.minusAndDeterminize(checkIndexAutomaton, restrictedIndices.getAutomaton());
             }
             if (false == Operations.isEmpty(checkIndexAutomaton)) {
                 Automaton allowedIndexPrivilegesAutomaton = null;
                 for (var indexAndPrivilegeAutomaton : indexGroupAutomatons.entrySet()) {
-                    if (Operations.subsetOf(checkIndexAutomaton, indexAndPrivilegeAutomaton.getValue())) {
+                    if (Automatons.subsetOf(checkIndexAutomaton, indexAndPrivilegeAutomaton.getValue())) {
                         if (allowedIndexPrivilegesAutomaton != null) {
-                            allowedIndexPrivilegesAutomaton = Automatons.unionAndMinimize(
+                            allowedIndexPrivilegesAutomaton = Automatons.unionAndDeterminize(
                                 Arrays.asList(allowedIndexPrivilegesAutomaton, indexAndPrivilegeAutomaton.getKey())
                             );
                         } else {
@@ -301,7 +301,7 @@ public boolean checkResourcePrivileges(
                 for (String privilege : checkForPrivileges) {
                     IndexPrivilege indexPrivilege = IndexPrivilege.get(Collections.singleton(privilege));
                     if (allowedIndexPrivilegesAutomaton != null
-                        && Operations.subsetOf(indexPrivilege.getAutomaton(), allowedIndexPrivilegesAutomaton)) {
+                        && Automatons.subsetOf(indexPrivilege.getAutomaton(), allowedIndexPrivilegesAutomaton)) {
                         if (resourcePrivilegesMapBuilder != null) {
                             resourcePrivilegesMapBuilder.addResourcePrivilege(forIndexPattern, privilege, Boolean.TRUE);
                         }
@@ -342,7 +342,7 @@ public Automaton allowedActionsMatcher(String index) {
                 automatonList.add(group.privilege.getAutomaton());
             }
         }
-        return automatonList.isEmpty() ? Automatons.EMPTY : Automatons.unionAndMinimize(automatonList);
+        return automatonList.isEmpty() ? Automatons.EMPTY : Automatons.unionAndDeterminize(automatonList);
     }
 
     /**
@@ -704,7 +704,7 @@ private Map<Automaton, Automaton> indexGroupAutomatons(boolean combine) {
             Automaton indexAutomaton = group.getIndexMatcherAutomaton();
             allAutomatons.compute(
                 group.privilege().getAutomaton(),
-                (key, value) -> value == null ? indexAutomaton : Automatons.unionAndMinimize(List.of(value, indexAutomaton))
+                (key, value) -> value == null ? indexAutomaton : Automatons.unionAndDeterminize(List.of(value, indexAutomaton))
             );
             if (combine) {
                 List<Tuple<Automaton, Automaton>> combinedAutomatons = new ArrayList<>();
@@ -714,7 +714,7 @@ private Map<Automaton, Automaton> indexGroupAutomatons(boolean combine) {
                         group.privilege().getAutomaton()
                     );
                     if (Operations.isEmpty(intersectingPrivileges) == false) {
-                        Automaton indexPatternAutomaton = Automatons.unionAndMinimize(
+                        Automaton indexPatternAutomaton = Automatons.unionAndDeterminize(
                             List.of(indexAndPrivilegeAutomatons.getValue(), indexAutomaton)
                         );
                         combinedAutomatons.add(new Tuple<>(intersectingPrivileges, indexPatternAutomaton));
@@ -723,7 +723,7 @@ private Map<Automaton, Automaton> indexGroupAutomatons(boolean combine) {
                 combinedAutomatons.forEach(
                     automatons -> allAutomatons.compute(
                         automatons.v1(),
-                        (key, value) -> value == null ? automatons.v2() : Automatons.unionAndMinimize(List.of(value, automatons.v2()))
+                        (key, value) -> value == null ? automatons.v2() : Automatons.unionAndDeterminize(List.of(value, automatons.v2()))
                     )
                 );
             }
@@ -768,7 +768,7 @@ public Group(
                 this.indexNameMatcher = StringMatcher.of(indices).and(name -> restrictedIndices.isRestricted(name) == false);
                 this.indexNameAutomaton = () -> indexNameAutomatonMemo.computeIfAbsent(
                     indices,
-                    k -> Automatons.minusAndMinimize(Automatons.patterns(indices), restrictedIndices.getAutomaton())
+                    k -> Automatons.minusAndDeterminize(Automatons.patterns(indices), restrictedIndices.getAutomaton())
                 );
             }
             this.fieldPermissions = Objects.requireNonNull(fieldPermissions);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
index ea32ba13ae576..e4d283aba75a3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
@@ -212,7 +212,7 @@ public IsResourceAuthorizedPredicate allowedIndicesMatcher(String action) {
     public Automaton allowedActionsMatcher(String index) {
         final Automaton allowedMatcher = baseRole.allowedActionsMatcher(index);
         final Automaton limitedByMatcher = limitedByRole.allowedActionsMatcher(index);
-        return Automatons.intersectAndMinimize(allowedMatcher, limitedByMatcher);
+        return Automatons.intersectAndDeterminize(allowedMatcher, limitedByMatcher);
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java
index 2960c5aaa53e7..0d8880c33720b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java
@@ -61,13 +61,15 @@
  */
 public class RemoteClusterPermissions implements NamedWriteable, ToXContentObject {
 
+    public static final TransportVersion ROLE_REMOTE_CLUSTER_PRIVS = TransportVersions.V_8_15_0;
+
     public static final String NAME = "remote_cluster_permissions";
     private static final Logger logger = LogManager.getLogger(RemoteClusterPermissions.class);
     private final List<RemoteClusterPermissionGroup> remoteClusterPermissionGroups;
 
     // package private non-final for testing
     static Map<TransportVersion, Set<String>> allowedRemoteClusterPermissions = Map.of(
-        TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS,
+        ROLE_REMOTE_CLUSTER_PRIVS,
         Set.of(ClusterPrivilegeResolver.MONITOR_ENRICH.name())
     );
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
index 7174b2f616c2a..f4df99dcefea4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
@@ -57,7 +57,7 @@
 
 import static java.util.Map.entry;
 import static org.elasticsearch.xpack.core.security.support.Automatons.patterns;
-import static org.elasticsearch.xpack.core.security.support.Automatons.unionAndMinimize;
+import static org.elasticsearch.xpack.core.security.support.Automatons.unionAndDeterminize;
 
 /**
  * The name of an index related action always being with `indices:` followed by a sequence of slash-separated terms
@@ -110,7 +110,7 @@ public final class IndexPrivilege extends Privilege {
     private static final Automaton DELETE_AUTOMATON = patterns("indices:data/write/delete*", "indices:data/write/bulk*");
     private static final Automaton WRITE_AUTOMATON = patterns("indices:data/write/*", TransportAutoPutMappingAction.TYPE.name());
     private static final Automaton MONITOR_AUTOMATON = patterns("indices:monitor/*");
-    private static final Automaton MANAGE_AUTOMATON = unionAndMinimize(
+    private static final Automaton MANAGE_AUTOMATON = unionAndDeterminize(
         Arrays.asList(
             MONITOR_AUTOMATON,
             patterns("indices:admin/*", TransportFieldCapabilitiesAction.NAME + "*", GetRollupIndexCapsAction.NAME + "*")
@@ -303,7 +303,7 @@ private static IndexPrivilege resolve(Set<String> name) {
         if (actions.isEmpty() == false) {
             automata.add(patterns(actions));
         }
-        return new IndexPrivilege(name, unionAndMinimize(automata));
+        return new IndexPrivilege(name, unionAndDeterminize(automata));
     }
 
     static Map<String, IndexPrivilege> values() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java
index 68e3f11751aac..7434128f03129 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/Privilege.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.core.security.authz.privilege;
 
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 
@@ -90,7 +89,7 @@ public static <T extends Privilege> SortedMap<String, T> sortByAccessLevel(Map<S
         privileges.forEach(
             (name, priv) -> subsetCount.put(
                 name,
-                privileges.values().stream().filter(p2 -> p2 != priv && Operations.subsetOf(priv.automaton, p2.automaton)).count()
+                privileges.values().stream().filter(p2 -> p2 != priv && Automatons.subsetOf(priv.automaton, p2.automaton)).count()
             )
         );
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
index 0028508e87f32..5fb753ab55aab 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
@@ -152,8 +152,11 @@ static RoleDescriptor kibanaSystem(String name) {
                 // Data telemetry reads mappings, metadata and stats of indices
                 RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("view_index_metadata", "monitor").build(),
                 // Endpoint diagnostic information. Kibana reads from these indices to send
-                // telemetry
-                RoleDescriptor.IndicesPrivileges.builder().indices(".logs-endpoint.diagnostic.collection-*").privileges("read").build(),
+                // telemetry and also creates the index when policies are first created
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".logs-endpoint.diagnostic.collection-*")
+                    .privileges("read", "create_index")
+                    .build(),
                 // Fleet secrets. Kibana can only write to this index.
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".fleet-secrets*")
@@ -277,17 +280,19 @@ static RoleDescriptor kibanaSystem(String name) {
                     )
                     .build(),
                 // Endpoint specific action responses. Kibana reads and writes (for third party
-                // agents) to the index
-                // to display action responses to the user.
+                // agents) to the index to display action responses to the user.
+                // `create_index`: is necessary in order to ensure that the DOT datastream index is
+                // created by Kibana in order to avoid errors on the Elastic Defend side when streaming
+                // documents to it.
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".logs-endpoint.action.responses-*")
-                    .privileges("auto_configure", "read", "write")
+                    .privileges("auto_configure", "read", "write", "create_index")
                     .build(),
                 // Endpoint specific actions. Kibana reads and writes to this index to track new
                 // actions and display them.
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".logs-endpoint.actions-*")
-                    .privileges("auto_configure", "read", "write")
+                    .privileges("auto_configure", "read", "write", "create_index")
                     .build(),
                 // Legacy Osquery manager specific action responses. Kibana reads from these to
                 // display responses to the user.
@@ -475,7 +480,7 @@ static RoleDescriptor kibanaSystem(String name) {
                 RoleDescriptor.IndicesPrivileges.builder().indices(".slo-observability.*").privileges("all").build(),
                 // Endpoint heartbeat. Kibana reads from these to determine metering/billing for
                 // endpoints.
-                RoleDescriptor.IndicesPrivileges.builder().indices(".logs-endpoint.heartbeat-*").privileges("read").build(),
+                RoleDescriptor.IndicesPrivileges.builder().indices(".logs-endpoint.heartbeat-*").privileges("read", "create_index").build(),
                 // For connectors telemetry. Will be removed once we switched to connectors API
                 RoleDescriptor.IndicesPrivileges.builder().indices(".elastic-connectors*").privileges("read").build() },
             null,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
index a6347d8b7ec77..d3790ea64ba4b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
@@ -9,9 +9,10 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
+import org.apache.lucene.util.automaton.StatePair;
+import org.apache.lucene.util.automaton.Transition;
 import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.cache.CacheBuilder;
 import org.elasticsearch.common.settings.Setting;
@@ -20,6 +21,7 @@
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.TimeValue;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -110,7 +112,7 @@ public static Automaton patterns(Collection<String> patterns) {
 
     private static Automaton buildAutomaton(Collection<String> patterns) {
         if (patterns.size() == 1) {
-            return minimize(pattern(patterns.iterator().next()));
+            return determinize(pattern(patterns.iterator().next()));
         }
 
         final Function<Collection<String>, Automaton> build = strings -> {
@@ -119,7 +121,7 @@ private static Automaton buildAutomaton(Collection<String> patterns) {
                 final Automaton patternAutomaton = pattern(pattern);
                 automata.add(patternAutomaton);
             }
-            return unionAndMinimize(automata);
+            return unionAndDeterminize(automata);
         };
 
         // We originally just compiled each automaton separately and then unioned them all.
@@ -186,7 +188,7 @@ private static Automaton buildAutomaton(Collection<String> patterns) {
         if (misc.isEmpty() == false) {
             automata.add(build.apply(misc));
         }
-        return unionAndMinimize(automata);
+        return unionAndDeterminize(automata);
     }
 
     /**
@@ -223,7 +225,10 @@ private static Automaton buildAutomaton(String pattern) {
                 );
             }
             String regex = pattern.substring(1, pattern.length() - 1);
-            return new RegExp(regex).toAutomaton();
+            return Operations.determinize(
+                new RegExp(regex, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton(),
+                DEFAULT_DETERMINIZE_WORK_LIMIT
+            );
         } else if (pattern.equals("*")) {
             return MATCH_ALL;
         } else {
@@ -269,26 +274,26 @@ static Automaton wildcard(String text) {
             }
             i += length;
         }
-        return concatenate(automata);
+        return Operations.determinize(concatenate(automata), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
-    public static Automaton unionAndMinimize(Collection<Automaton> automata) {
+    public static Automaton unionAndDeterminize(Collection<Automaton> automata) {
         Automaton res = automata.size() == 1 ? automata.iterator().next() : union(automata);
-        return minimize(res);
+        return determinize(res);
     }
 
-    public static Automaton minusAndMinimize(Automaton a1, Automaton a2) {
+    public static Automaton minusAndDeterminize(Automaton a1, Automaton a2) {
         Automaton res = minus(a1, a2, maxDeterminizedStates);
-        return minimize(res);
+        return determinize(res);
     }
 
-    public static Automaton intersectAndMinimize(Automaton a1, Automaton a2) {
+    public static Automaton intersectAndDeterminize(Automaton a1, Automaton a2) {
         Automaton res = intersection(a1, a2);
-        return minimize(res);
+        return determinize(res);
     }
 
-    private static Automaton minimize(Automaton automaton) {
-        return MinimizationOperations.minimize(automaton, maxDeterminizedStates);
+    private static Automaton determinize(Automaton automaton) {
+        return Operations.determinize(automaton, maxDeterminizedStates);
     }
 
     public static Predicate<String> predicate(String... patterns) {
@@ -329,7 +334,8 @@ private static Predicate<String> predicate(Automaton automaton, final String toS
         } else if (automaton == EMPTY) {
             return Predicates.never();
         }
-        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, maxDeterminizedStates);
+        automaton = Operations.determinize(automaton, maxDeterminizedStates);
+        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton);
         return new Predicate<String>() {
             @Override
             public boolean test(String s) {
@@ -368,4 +374,72 @@ static List<String> getPatterns(Automaton automaton) {
             throw new IllegalArgumentException("recordPatterns is set to false");
         }
     }
+
+    /**
+     * Returns true if the language of <code>a1</code> is a subset of the language of <code>a2</code>.
+     * Both automata must be determinized and must have no dead states.
+     *
+     * <p>Complexity: quadratic in number of states.
+     * Copied of Lucene's AutomatonTestUtil
+     */
+    public static boolean subsetOf(Automaton a1, Automaton a2) {
+        if (a1.isDeterministic() == false) {
+            throw new IllegalArgumentException("a1 must be deterministic");
+        }
+        if (a2.isDeterministic() == false) {
+            throw new IllegalArgumentException("a2 must be deterministic");
+        }
+        assert Operations.hasDeadStatesFromInitial(a1) == false;
+        assert Operations.hasDeadStatesFromInitial(a2) == false;
+        if (a1.getNumStates() == 0) {
+            // Empty language is alwyas a subset of any other language
+            return true;
+        } else if (a2.getNumStates() == 0) {
+            return Operations.isEmpty(a1);
+        }
+
+        // TODO: cutover to iterators instead
+        Transition[][] transitions1 = a1.getSortedTransitions();
+        Transition[][] transitions2 = a2.getSortedTransitions();
+        ArrayDeque<StatePair> worklist = new ArrayDeque<>();
+        HashSet<StatePair> visited = new HashSet<>();
+        StatePair p = new StatePair(0, 0);
+        worklist.add(p);
+        visited.add(p);
+        while (worklist.size() > 0) {
+            p = worklist.removeFirst();
+            if (a1.isAccept(p.s1) && a2.isAccept(p.s2) == false) {
+                return false;
+            }
+            Transition[] t1 = transitions1[p.s1];
+            Transition[] t2 = transitions2[p.s2];
+            for (int n1 = 0, b2 = 0; n1 < t1.length; n1++) {
+                while (b2 < t2.length && t2[b2].max < t1[n1].min) {
+                    b2++;
+                }
+                int min1 = t1[n1].min, max1 = t1[n1].max;
+
+                for (int n2 = b2; n2 < t2.length && t1[n1].max >= t2[n2].min; n2++) {
+                    if (t2[n2].min > min1) {
+                        return false;
+                    }
+                    if (t2[n2].max < Character.MAX_CODE_POINT) {
+                        min1 = t2[n2].max + 1;
+                    } else {
+                        min1 = Character.MAX_CODE_POINT;
+                        max1 = Character.MIN_CODE_POINT;
+                    }
+                    StatePair q = new StatePair(t1[n1].dest, t2[n2].dest);
+                    if (visited.contains(q) == false) {
+                        worklist.add(q);
+                        visited.add(q);
+                    }
+                }
+                if (min1 <= max1) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/SecurityMigrationTaskParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/SecurityMigrationTaskParams.java
index 14cc4d3d6f5b9..e69a058914634 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/SecurityMigrationTaskParams.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/SecurityMigrationTaskParams.java
@@ -48,7 +48,7 @@ public SecurityMigrationTaskParams(int migrationVersion, boolean migrationNeeded
 
     public SecurityMigrationTaskParams(StreamInput in) throws IOException {
         this.migrationVersion = in.readInt();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.SECURITY_MIGRATIONS_MIGRATION_NEEDED_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.migrationNeeded = in.readBoolean();
         } else {
             this.migrationNeeded = true;
@@ -58,7 +58,7 @@ public SecurityMigrationTaskParams(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeInt(migrationVersion);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.SECURITY_MIGRATIONS_MIGRATION_NEEDED_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeBoolean(migrationNeeded);
         }
     }
@@ -70,7 +70,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ADD_METADATA_FLATTENED_TO_ROLES;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java
index 0ada92bbb1e68..186cd81537909 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java
@@ -106,9 +106,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         {
             builder.field(SNAPSHOT_NAME.getPreferredName(), snapshotName);
             if (snapshotStartTimestamp != null) {
-                builder.timeField(START_TIMESTAMP.getPreferredName(), "start_time_string", snapshotStartTimestamp);
+                builder.timestampFieldsFromUnixEpochMillis(START_TIMESTAMP.getPreferredName(), "start_time_string", snapshotStartTimestamp);
             }
-            builder.timeField(TIMESTAMP.getPreferredName(), "time_string", snapshotFinishTimestamp);
+            builder.timestampFieldsFromUnixEpochMillis(TIMESTAMP.getPreferredName(), "time_string", snapshotFinishTimestamp);
             if (Objects.nonNull(details)) {
                 builder.field(DETAILS.getPreferredName(), details);
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java
index c3c70e595eb75..ea52930f4ae84 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java
@@ -157,7 +157,7 @@ public boolean equals(Object obj) {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(policy.getId());
         builder.field(SnapshotLifecyclePolicyMetadata.VERSION.getPreferredName(), version);
-        builder.timeField(
+        builder.timestampFieldsFromUnixEpochMillis(
             SnapshotLifecyclePolicyMetadata.MODIFIED_DATE_MILLIS.getPreferredName(),
             SnapshotLifecyclePolicyMetadata.MODIFIED_DATE.getPreferredName(),
             modifiedDate
@@ -169,7 +169,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (lastFailure != null) {
             builder.field(SnapshotLifecyclePolicyMetadata.LAST_FAILURE.getPreferredName(), lastFailure);
         }
-        builder.timeField(
+        builder.timestampFieldsFromUnixEpochMillis(
             SnapshotLifecyclePolicyMetadata.NEXT_EXECUTION_MILLIS.getPreferredName(),
             SnapshotLifecyclePolicyMetadata.NEXT_EXECUTION.getPreferredName(),
             policy.calculateNextExecution(modifiedDate, Clock.systemUTC())
@@ -249,7 +249,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(NAME.getPreferredName(), snapshotId.getName());
             builder.field(UUID.getPreferredName(), snapshotId.getUUID());
             builder.field(STATE.getPreferredName(), state);
-            builder.timeField(START_TIME.getPreferredName(), "start_time", startTime);
+            builder.timestampFieldsFromUnixEpochMillis(START_TIME.getPreferredName(), "start_time", startTime);
             if (failure != null) {
                 builder.field(FAILURE.getPreferredName(), failure);
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
index 672578787762e..dfaaa48f1e2cb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
@@ -192,7 +192,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(POLICY.getPreferredName(), policy);
         builder.field(HEADERS.getPreferredName(), headers);
         builder.field(VERSION.getPreferredName(), version);
-        builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate);
+        builder.timestampFieldsFromUnixEpochMillis(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate);
         if (Objects.nonNull(lastSuccess)) {
             builder.field(LAST_SUCCESS.getPreferredName(), lastSuccess);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java
index ee077e5140606..06ff971ecf890 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java
@@ -134,7 +134,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             .field("subject_dn", subjectDn)
             .field("serial_number", serialNumber)
             .field("has_private_key", hasPrivateKey)
-            .timeField("expiry", expiry);
+            .timestampField("expiry", expiry);
         if (Strings.hasLength(issuer)) {
             builder.field("issuer", issuer);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/rest/RestGetCertificateInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/rest/RestGetCertificateInfoAction.java
index 38e59a2e34df8..cf9dbd71b98fb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/rest/RestGetCertificateInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/rest/RestGetCertificateInfoAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.core.ssl.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -32,7 +31,7 @@ public class RestGetCertificateInfoAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(GET, "/_ssl/certificates").replaces(GET, "/_xpack/ssl/certificates", RestApiVersion.V_7).build());
+        return List.of(new Route(GET, "/_ssl/certificates"));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java
index b4952373dfdd3..92568c4f31c18 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOBooleanSupplier;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
@@ -69,6 +70,11 @@ public AttributeSource attributes() {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public IOBooleanSupplier prepareSeekExact(BytesRef bytesRef) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public boolean seekExact(BytesRef text) throws IOException {
         throw new UnsupportedOperationException();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java
index 2828a46a28b8c..aa256940daa9b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java
@@ -93,14 +93,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(TransformField.CHECKPOINT_PROGRESS.getPreferredName(), checkpointProgress);
         }
         if (timestampMillis > 0) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 TransformField.TIMESTAMP_MILLIS.getPreferredName(),
                 TransformField.TIMESTAMP.getPreferredName(),
                 timestampMillis
             );
         }
         if (timeUpperBoundMillis > 0) {
-            builder.timeField(
+            builder.timestampFieldsFromUnixEpochMillis(
                 TransformField.TIME_UPPER_BOUND_MILLIS.getPreferredName(),
                 TransformField.TIME_UPPER_BOUND.getPreferredName(),
                 timeUpperBoundMillis
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java
index c4530c535cbcf..a6e365b793d93 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java
@@ -217,10 +217,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(OPERATIONS_BEHIND, operationsBehind);
         }
         if (changesLastDetectedAt != null) {
-            builder.timeField(CHANGES_LAST_DETECTED_AT, CHANGES_LAST_DETECTED_AT_HUMAN, changesLastDetectedAt.toEpochMilli());
+            builder.timestampFieldsFromUnixEpochMillis(
+                CHANGES_LAST_DETECTED_AT,
+                CHANGES_LAST_DETECTED_AT_HUMAN,
+                changesLastDetectedAt.toEpochMilli()
+            );
         }
         if (lastSearchTime != null) {
-            builder.timeField(LAST_SEARCH_TIME, LAST_SEARCH_TIME_HUMAN, lastSearchTime.toEpochMilli());
+            builder.timestampFieldsFromUnixEpochMillis(LAST_SEARCH_TIME, LAST_SEARCH_TIME_HUMAN, lastSearchTime.toEpochMilli());
         }
         builder.endObject();
         return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java
index fb782bdae0068..d8972dcf6a6be 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java
@@ -450,7 +450,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
                 builder.field(TransformField.VERSION.getPreferredName(), transformVersion);
             }
             if (createTime != null) {
-                builder.timeField(
+                builder.timestampFieldsFromUnixEpochMillis(
                     TransformField.CREATE_TIME.getPreferredName(),
                     TransformField.CREATE_TIME.getPreferredName() + "_string",
                     createTime.toEpochMilli()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java
index 5697e1793f0b0..451cfd89f31af 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java
@@ -90,7 +90,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
         builder.field(COUNT, count);
         if (firstOccurrence != null) {
-            builder.timeField(FIRST_OCCURRENCE, FIRST_OCCURRENCE_HUMAN_READABLE, firstOccurrence.toEpochMilli());
+            builder.timestampFieldsFromUnixEpochMillis(FIRST_OCCURRENCE, FIRST_OCCURRENCE_HUMAN_READABLE, firstOccurrence.toEpochMilli());
         }
         return builder.endObject();
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java
index 4da5d46e82fa6..a7633ed0fa1a1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java
@@ -71,8 +71,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field("watch_id", watchId);
         builder.field("watch_record_id", watchRecordId);
-        builder.timeField("triggered_time", triggeredTime);
-        builder.timeField("execution_time", executionTime);
+        builder.timestampField("triggered_time", triggeredTime);
+        builder.timestampField("execution_time", executionTime);
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java
index 2b80c32f3c327..49d0566dffeaa 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java
@@ -108,8 +108,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field("watch_id", watchId);
         builder.field("watch_record_id", watchRecordId);
-        builder.timeField("triggered_time", triggeredTime);
-        builder.timeField("execution_time", executionTime);
+        builder.timestampField("triggered_time", triggeredTime);
+        builder.timestampField("execution_time", executionTime);
         builder.field("execution_phase", phase);
         if (executedActions != null) {
             builder.array("executed_actions", executedActions);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java
index f1d046f09b0f7..c6a22ec289be9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java
@@ -37,7 +37,7 @@ public Request(TimeValue masterNodeTimeout) {
         }
 
         public static Request readFrom(StreamInput in) throws IOException {
-            if (in.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 return new Request(in);
             } else {
                 return new Request(TimeValue.THIRTY_SECONDS);
@@ -50,7 +50,7 @@ private Request(StreamInput in) throws IOException {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 super.writeTo(out);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java
index 19cc83cda2dbb..7b0bd8a8108e9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.UpdateForV9;
 
 import java.io.IOException;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 public class UpdateWatcherSettingsAction extends ActionType<AcknowledgedResponse> {
 
@@ -34,6 +34,16 @@ public class UpdateWatcherSettingsAction extends ActionType<AcknowledgedResponse
         IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS
     );
 
+    public static final Set<String> ALLOWED_SETTINGS_PREFIXES = Set.of(
+        IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX,
+        IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX,
+        IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX
+    );
+
+    public static final Set<String> EXPLICITLY_DENIED_SETTINGS = Set.of(
+        IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "._tier_preference"
+    );
+
     public UpdateWatcherSettingsAction() {
         super(NAME);
     }
@@ -47,7 +57,7 @@ public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, Map<String, Ob
         }
 
         public static Request readFrom(StreamInput in) throws IOException {
-            if (in.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 return new Request(in);
             } else {
                 return new Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, in);
@@ -67,7 +77,7 @@ private Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, StreamInput i
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 super.writeTo(out);
             }
             out.writeGenericMap(this.settings);
@@ -79,13 +89,25 @@ public Map<String, Object> settings() {
 
         @Override
         public ActionRequestValidationException validate() {
-            Set<String> forbiddenSettings = Sets.difference(settings.keySet(), ALLOWED_SETTING_KEYS);
-            if (forbiddenSettings.size() > 0) {
+            Set<String> forbiddenSettings = settings.keySet()
+                .stream()
+                .filter(
+                    setting -> (ALLOWED_SETTING_KEYS.contains(setting) == false
+                        && ALLOWED_SETTINGS_PREFIXES.stream().noneMatch(prefix -> setting.startsWith(prefix + ".")))
+                        || EXPLICITLY_DENIED_SETTINGS.contains(setting)
+                )
+                .collect(Collectors.toSet());
+
+            if (forbiddenSettings.isEmpty() == false) {
                 return ValidateActions.addValidationError(
                     "illegal settings: "
                         + forbiddenSettings
                         + ", these settings may not be configured. Only the following settings may be configured: "
-                        + ALLOWED_SETTING_KEYS,
+                        + ALLOWED_SETTING_KEYS
+                        + ", "
+                        + ALLOWED_SETTINGS_PREFIXES.stream().map(s -> s + ".*").collect(Collectors.toSet())
+                        + " excluding the following explicitly denied settings: "
+                        + EXPLICITLY_DENIED_SETTINGS,
                     null
                 );
             }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
index e39ddc170c0a9..54390365c62af 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -400,14 +401,14 @@ public void onFailure(Exception e) {
         try (Engine.Searcher searcher = restoredShard.acquireSearcher("test")) {
             assertEquals(searcher.getIndexReader().maxDoc(), seqNoStats.getLocalCheckpoint());
             TopDocs search = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
-            assertEquals(searcher.getIndexReader().numDocs(), search.totalHits.value);
+            assertEquals(searcher.getIndexReader().numDocs(), search.totalHits.value());
             search = searcher.search(
                 new MatchAllDocsQuery(),
                 Integer.MAX_VALUE,
                 new Sort(new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG)),
                 false
             );
-            assertEquals(searcher.getIndexReader().numDocs(), search.totalHits.value);
+            assertEquals(searcher.getIndexReader().numDocs(), search.totalHits.value());
             long previous = -1;
             for (ScoreDoc doc : search.scoreDocs) {
                 FieldDoc fieldDoc = (FieldDoc) doc;
@@ -430,8 +431,9 @@ public void onFailure(Exception e) {
             assertEquals(original.exists(), restored.exists());
 
             if (original.exists()) {
-                Document document = original.docIdAndVersion().reader.document(original.docIdAndVersion().docId);
-                Document restoredDocument = restored.docIdAndVersion().reader.document(restored.docIdAndVersion().docId);
+                StoredFields storedFields = original.docIdAndVersion().reader.storedFields();
+                Document document = storedFields.document(original.docIdAndVersion().docId);
+                Document restoredDocument = storedFields.document(restored.docIdAndVersion().docId);
                 for (IndexableField field : document) {
                     assertEquals(document.get(field.name()), restoredDocument.get(field.name()));
                 }
@@ -470,7 +472,7 @@ public IndexShard reindex(DirectoryReader reader, MappingMetadata mapping) throw
                 for (int i = 0; i < leafReader.maxDoc(); i++) {
                     if (liveDocs == null || liveDocs.get(i)) {
                         rootFieldsVisitor.reset();
-                        leafReader.document(i, rootFieldsVisitor);
+                        leafReader.storedFields().document(i, rootFieldsVisitor);
                         rootFieldsVisitor.postProcess(targetShard.mapperService()::fieldType);
                         String id = rootFieldsVisitor.id();
                         BytesReference source = rootFieldsVisitor.source();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotTests.java
index 65d057408f8bd..8433f38e40a0c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotTests.java
@@ -32,6 +32,7 @@
 import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
 import org.apache.lucene.index.StandardDirectoryReader;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.IndexSearcher;
@@ -108,8 +109,10 @@ DirectoryReader wrapReader(DirectoryReader reader) throws IOException {
                         logger.warn(snapReader + " " + reader);
                         assertEquals(snapReader.maxDoc(), reader.maxDoc());
                         assertEquals(snapReader.numDocs(), reader.numDocs());
+                        StoredFields snapStoredFields = snapReader.storedFields();
+                        StoredFields storedFields = reader.storedFields();
                         for (int i = 0; i < snapReader.maxDoc(); i++) {
-                            assertEquals(snapReader.document(i).get("_source"), reader.document(i).get("_source"));
+                            assertEquals(snapStoredFields.document(i).get("_source"), storedFields.document(i).get("_source"));
                         }
                         for (LeafReaderContext ctx : snapReader.leaves()) {
                             if (ctx.reader() instanceof SegmentReader) {
@@ -188,12 +191,14 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo,
             try (DirectoryReader snapReader = DirectoryReader.open(wrappedDir)) {
                 assertEquals(snapReader.maxDoc(), 3);
                 assertEquals(snapReader.numDocs(), 2);
+                StoredFields snapStoredFields = snapReader.storedFields();
+                StoredFields storedFields = reader.storedFields();
                 for (int i = 0; i < 3; i++) {
-                    assertEquals(snapReader.document(i).get("src"), reader.document(i).get("src"));
+                    assertEquals(snapStoredFields.document(i).get("src"), storedFields.document(i).get("src"));
                 }
                 IndexSearcher searcher = newSearcher(snapReader);
                 TopDocs id = searcher.search(new TermQuery(new Term("id", "1")), 10);
-                assertEquals(0, id.totalHits.value);
+                assertEquals(0, id.totalHits.value());
             }
 
             targetDir = newDirectory(targetDir);
@@ -321,7 +326,7 @@ public boolean keepFullyDeletedSegment(IOSupplier<CodecReader> readerIOSupplier)
                 try (DirectoryReader snapReader = DirectoryReader.open(wrappedDir)) {
                     assertEquals(snapReader.maxDoc(), 1);
                     assertEquals(snapReader.numDocs(), 1);
-                    assertEquals("3", snapReader.document(0).getField("rank").stringValue());
+                    assertEquals("3", snapReader.storedFields().document(0).getField("rank").stringValue());
                 }
                 try (IndexReader writerReader = DirectoryReader.open(writer)) {
                     assertEquals(writerReader.maxDoc(), 2);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
index 918976c0d3db8..1f2c89c473a62 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
@@ -637,10 +637,15 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
 
     @Override
     public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
-        return filterPlugins(MapperPlugin.class).stream()
+        Map<String, MetadataFieldMapper.TypeParser> pluginsMetadataMappers = filterPlugins(MapperPlugin.class).stream()
             .map(MapperPlugin::getMetadataMappers)
             .flatMap(map -> map.entrySet().stream())
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        // the xpack plugin itself exposes a metadata mapper so let's include it as well
+        Map<String, MetadataFieldMapper.TypeParser> metadataMappersIncludingXPackPlugin = new HashMap<>(pluginsMetadataMappers);
+        metadataMappersIncludingXPackPlugin.putAll(super.getMetadataMappers());
+        return metadataMappersIncludingXPackPlugin;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
index 15958e9396d81..afb17644303bb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
@@ -308,8 +308,8 @@ public void onFailure(Exception e) {
         verify(indicesClient, Mockito.only()).rolloverIndex(requestCaptor.capture(), Mockito.any());
 
         RolloverRequest request = requestCaptor.getValue();
-        assertThat(request.indicesOptions().failureStoreOptions().includeFailureIndices(), equalTo(failureStoreIndex));
-        assertThat(request.indicesOptions().failureStoreOptions().includeRegularIndices(), not(equalTo(failureStoreIndex)));
+        assertThat(request.indicesOptions().includeFailureIndices(), equalTo(failureStoreIndex));
+        assertThat(request.indicesOptions().includeRegularIndices(), not(equalTo(failureStoreIndex)));
     }
 
     public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java
index b84aaa2bcfc1b..46f10928cad08 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.core.inference.results;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.AbstractChunkedBWCSerializationTestCase;
@@ -16,8 +17,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.TransportVersions.ML_RERANK_DOC_OPTIONAL;
-
 public class RankedDocsResultsTests extends AbstractChunkedBWCSerializationTestCase<RankedDocsResults> {
 
     @Override
@@ -47,7 +46,7 @@ protected RankedDocsResults mutateInstance(RankedDocsResults instance) throws IO
 
     @Override
     protected RankedDocsResults mutateInstanceForVersion(RankedDocsResults instance, TransportVersion fromVersion) {
-        if (fromVersion.onOrAfter(ML_RERANK_DOC_OPTIONAL)) {
+        if (fromVersion.onOrAfter(TransportVersions.V_8_15_0)) {
             return instance;
         } else {
             var compatibleDocs = rankedDocsNullStringToEmpty(instance.getRankedDocs());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
index d60bbc6cc7713..1b9d4ebb100b2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
@@ -562,7 +562,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion
                     RESULTS_FIELD
                 )
             );
-        } else if (version.before(TransportVersions.NODE_STATS_INGEST_BYTES)) {
+        } else if (version.before(TransportVersions.V_8_15_0)) {
             // added ByteStats to IngestStats.PipelineStat
             return new Response(
                 new QueryPage<>(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
index 2e4689de787b3..05266e58964c9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
@@ -252,7 +252,7 @@ protected Request mutateInstanceForVersion(Request instance, TransportVersion ve
             r.setHighPriority(instance.isHighPriority());
             r.setPrefixType(TrainedModelPrefixStrings.PrefixType.NONE);
             return r;
-        } else if (version.before(TransportVersions.ML_CHUNK_INFERENCE_OPTION)) {
+        } else if (version.before(TransportVersions.V_8_15_0)) {
             var r = new Request(
                 instance.getId(),
                 adjustedUpdate,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
index 730d994fc5e35..46fc8a36c2c2b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
@@ -67,7 +67,7 @@ public static Request createRandom() {
             request.setNumberOfAllocations(randomIntBetween(1, 8));
         }
         if (randomBoolean()) {
-            request.setQueueCapacity(randomIntBetween(1, 1000000));
+            request.setQueueCapacity(randomIntBetween(1, 100_000));
         }
         if (randomBoolean()) {
             request.setPriority(randomFrom(Priority.values()).toString());
@@ -168,7 +168,7 @@ public void testValidate_GivenQueueCapacityIsNegative() {
 
     public void testValidate_GivenQueueCapacityIsAtLimit() {
         Request request = createRandom();
-        request.setQueueCapacity(1_000_000);
+        request.setQueueCapacity(100_000);
 
         ActionRequestValidationException e = request.validate();
 
@@ -177,12 +177,12 @@ public void testValidate_GivenQueueCapacityIsAtLimit() {
 
     public void testValidate_GivenQueueCapacityIsOverLimit() {
         Request request = createRandom();
-        request.setQueueCapacity(1_000_001);
+        request.setQueueCapacity(100_001);
 
         ActionRequestValidationException e = request.validate();
 
         assertThat(e, is(not(nullValue())));
-        assertThat(e.getMessage(), containsString("[queue_capacity] must be less than 1000000"));
+        assertThat(e.getMessage(), containsString("[queue_capacity] must be less than 100000"));
     }
 
     public void testValidate_GivenTimeoutIsNegative() {
@@ -234,6 +234,6 @@ public void testDefaults() {
         assertThat(request.getNumberOfAllocations(), nullValue());
         assertThat(request.computeNumberOfAllocations(), equalTo(1));
         assertThat(request.getThreadsPerAllocation(), equalTo(1));
-        assertThat(request.getQueueCapacity(), equalTo(1024));
+        assertThat(request.getQueueCapacity(), equalTo(10_000));
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
index 6a213d6f5e379..c3b6e0089b4ae 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
@@ -195,7 +195,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenNoS
         builder.addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STOPPED, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        assertThat(assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STARTED).isEmpty(), is(true));
+        assertThat(assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STARTED).isEmpty(), is(true));
     }
 
     public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenSingleStartedNode() {
@@ -203,7 +203,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenSin
         builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTED, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STARTED);
+        var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STARTED);
 
         assertThat(nodes, contains(new Tuple<>("node-1", 1)));
     }
@@ -213,7 +213,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenASh
         builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTED, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STOPPING);
+        var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STOPPING);
 
         assertThat(nodes, empty());
     }
@@ -223,7 +223,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenASh
         builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STOPPING, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STOPPING);
+        var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STOPPING);
 
         assertThat(nodes, contains(new Tuple<>("node-1", 1)));
     }
@@ -234,7 +234,7 @@ public void testSingleRequestWith2Nodes() {
         builder.addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STARTED);
+        var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STARTED);
         assertThat(nodes, hasSize(1));
         assertEquals(nodes.get(0).v2(), Integer.valueOf(1));
     }
@@ -248,7 +248,7 @@ public void testSelectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenMul
 
         final int selectionCount = 10000;
         final CountAccumulator countsPerNodeAccumulator = new CountAccumulator();
-        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(selectionCount, RoutingState.STARTED);
+        var nodes = assignment.selectRandomNodesWeighedOnAllocations(selectionCount, RoutingState.STARTED);
 
         assertThat(nodes, hasSize(3));
         assertThat(nodes.stream().mapToInt(Tuple::v2).sum(), equalTo(selectionCount));
@@ -269,7 +269,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenMul
         builder.addRoutingEntry("node-3", new RoutingInfo(0, 0, RoutingState.STARTED, ""));
         TrainedModelAssignment assignment = builder.build();
         final int selectionCount = 1000;
-        var nodeCounts = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(selectionCount, RoutingState.STARTED);
+        var nodeCounts = assignment.selectRandomNodesWeighedOnAllocations(selectionCount, RoutingState.STARTED);
         assertThat(nodeCounts, hasSize(3));
 
         var selectedNodes = new HashSet<String>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/WeightedTokensQueryBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/WeightedTokensQueryBuilderTests.java
index bb727204e2651..114ad90354c61 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/WeightedTokensQueryBuilderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/WeightedTokensQueryBuilderTests.java
@@ -279,7 +279,7 @@ private void assertCorrectLuceneQuery(String name, Query query, List<String> exp
             booleanClauses.size()
         );
         for (int i = 0; i < booleanClauses.size(); i++) {
-            Query clauseQuery = booleanClauses.get(i).getQuery();
+            Query clauseQuery = booleanClauses.get(i).query();
             assertTrue(name + " query " + query + " expected to be a BoostQuery", clauseQuery instanceof BoostQuery);
             // FeatureQuery is not visible so we check the String representation
             assertTrue(name + " query " + query + " expected to be a FeatureQuery", clauseQuery.toString().contains("FeatureQuery"));
@@ -353,8 +353,8 @@ protected void doAssertLuceneQuery(WeightedTokensQueryBuilder queryBuilder, Quer
         Class<?> boostQueryClass = FeatureField.newLinearQuery("", "", 1.0f).getClass();
 
         for (var clause : booleanQuery.clauses()) {
-            assertEquals(BooleanClause.Occur.SHOULD, clause.getOccur());
-            assertThat(clause.getQuery(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass)));
+            assertEquals(BooleanClause.Occur.SHOULD, clause.occur());
+            assertThat(clause.query(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass)));
         }
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
index 437f58449b4de..ab62073c12d80 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
@@ -43,6 +43,7 @@
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -69,7 +70,7 @@ public void testSerialization() throws IOException {
     public void testSerializationForCurrentVersion() throws Exception {
         final TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random());
         final boolean canIncludeRemoteIndices = version.onOrAfter(TransportVersions.V_8_8_0);
-        final boolean canIncludeRemoteCluster = version.onOrAfter(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS);
+        final boolean canIncludeRemoteCluster = version.onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS);
 
         final GetUserPrivilegesResponse original = randomResponse(canIncludeRemoteIndices, canIncludeRemoteCluster);
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
index 3d4d6106a7eaf..66e246d1c8a50 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
@@ -44,6 +44,7 @@
 import static java.util.Map.entry;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo;
 import static org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfoTests.randomRoleDescriptorsIntersection;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -1108,7 +1109,7 @@ public void testMaybeRewriteMetadataForApiKeyRoleDescriptorsWithRemoteCluster()
         final Authentication original = AuthenticationTestHelper.builder()
             .apiKey()
             .metadata(metadata)
-            .transportVersion(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS)
+            .transportVersion(ROLE_REMOTE_CLUSTER_PRIVS)
             .build();
 
         // pick a version before that of the authentication instance to force a rewrite
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java
index 8e1bc7af1bdc8..94430a4ed5bba 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java
@@ -45,10 +45,12 @@
 import java.util.Map;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.SECURITY_ROLE_DESCRIPTION;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.WORKFLOWS_RESTRICTION_VERSION;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomIndicesPrivileges;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomIndicesPrivilegesBuilder;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomRemoteClusterPermissions;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
@@ -589,9 +591,9 @@ public void testParsingFieldPermissionsUsesCache() throws IOException {
     public void testSerializationForCurrentVersion() throws Exception {
         final TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random());
         final boolean canIncludeRemoteIndices = version.onOrAfter(TransportVersions.V_8_8_0);
-        final boolean canIncludeRemoteClusters = version.onOrAfter(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS);
+        final boolean canIncludeRemoteClusters = version.onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS);
         final boolean canIncludeWorkflows = version.onOrAfter(WORKFLOWS_RESTRICTION_VERSION);
-        final boolean canIncludeDescription = version.onOrAfter(TransportVersions.SECURITY_ROLE_DESCRIPTION);
+        final boolean canIncludeDescription = version.onOrAfter(SECURITY_ROLE_DESCRIPTION);
         logger.info("Testing serialization with version {}", version);
         BytesStreamOutput output = new BytesStreamOutput();
         output.setTransportVersion(version);
@@ -667,9 +669,7 @@ public void testSerializationWithRemoteIndicesWithElderVersion() throws IOExcept
     }
 
     public void testSerializationWithRemoteClusterWithElderVersion() throws IOException {
-        final TransportVersion versionBeforeRemoteCluster = TransportVersionUtils.getPreviousVersion(
-            TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS
-        );
+        final TransportVersion versionBeforeRemoteCluster = TransportVersionUtils.getPreviousVersion(ROLE_REMOTE_CLUSTER_PRIVS);
         final TransportVersion version = TransportVersionUtils.randomVersionBetween(
             random(),
             TransportVersions.V_7_17_0,
@@ -815,9 +815,7 @@ public void testParseRoleWithRestrictionWhenAllowRestrictionIsTrue() throws IOEx
     }
 
     public void testSerializationWithDescriptionAndUnsupportedVersions() throws IOException {
-        final TransportVersion versionBeforeRoleDescription = TransportVersionUtils.getPreviousVersion(
-            TransportVersions.SECURITY_ROLE_DESCRIPTION
-        );
+        final TransportVersion versionBeforeRoleDescription = TransportVersionUtils.getPreviousVersion(SECURITY_ROLE_DESCRIPTION);
         final TransportVersion version = TransportVersionUtils.randomVersionBetween(
             random(),
             TransportVersions.V_7_17_0,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
index c40dd00e0e350..6fe271d1b05e3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
@@ -96,25 +96,25 @@ public void testSearch() throws Exception {
         );
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1));
         TopDocs result = indexSearcher.search(new MatchAllDocsQuery(), 1);
-        assertThat(result.totalHits.value, equalTo(1L));
+        assertThat(result.totalHits.value(), equalTo(1L));
         assertThat(result.scoreDocs[0].doc, equalTo(0));
 
         indexSearcher = newSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetCache, new TermQuery(new Term("field", "value2"))));
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1));
         result = indexSearcher.search(new MatchAllDocsQuery(), 1);
-        assertThat(result.totalHits.value, equalTo(1L));
+        assertThat(result.totalHits.value(), equalTo(1L));
         assertThat(result.scoreDocs[0].doc, equalTo(1));
 
         // this doc has been marked as deleted:
         indexSearcher = newSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetCache, new TermQuery(new Term("field", "value3"))));
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(0));
         result = indexSearcher.search(new MatchAllDocsQuery(), 1);
-        assertThat(result.totalHits.value, equalTo(0L));
+        assertThat(result.totalHits.value(), equalTo(0L));
 
         indexSearcher = newSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetCache, new TermQuery(new Term("field", "value4"))));
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1));
         result = indexSearcher.search(new MatchAllDocsQuery(), 1);
-        assertThat(result.totalHits.value, equalTo(1L));
+        assertThat(result.totalHits.value(), equalTo(1L));
         assertThat(result.scoreDocs[0].doc, equalTo(3));
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
index 560dee9b5843c..db250b16eab16 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
@@ -30,6 +30,7 @@
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
@@ -205,8 +206,9 @@ public void testKnnVectors() throws Exception {
         FloatVectorValues vectorValues = leafReader.getFloatVectorValues("fieldA");
         assertEquals(3, vectorValues.dimension());
         assertEquals(1, vectorValues.size());
-        assertEquals(0, vectorValues.nextDoc());
-        assertNotNull(vectorValues.vectorValue());
+        KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
+        assertEquals(0, iterator.nextDoc());
+        assertNotNull(vectorValues.vectorValue(iterator.index()));
 
         TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new float[] { 1.0f, 1.0f, 1.0f }, 5, null, Integer.MAX_VALUE);
         assertNotNull(topDocs);
@@ -215,7 +217,7 @@ public void testKnnVectors() throws Exception {
         // Check that we can't see fieldB
         assertNull(leafReader.getFloatVectorValues("fieldB"));
         topDocs = leafReader.searchNearestVectors("fieldB", new float[] { 1.0f, 1.0f, 1.0f }, 5, null, Integer.MAX_VALUE);
-        assertEquals(0, topDocs.totalHits.value);
+        assertEquals(0, topDocs.totalHits.value());
         assertEquals(0, topDocs.scoreDocs.length);
 
         TestUtil.checkReader(ir);
@@ -239,8 +241,9 @@ public void testKnnByteVectors() throws Exception {
         ByteVectorValues vectorValues = leafReader.getByteVectorValues("fieldA");
         assertEquals(3, vectorValues.dimension());
         assertEquals(1, vectorValues.size());
-        assertEquals(0, vectorValues.nextDoc());
-        assertNotNull(vectorValues.vectorValue());
+        KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
+        assertEquals(0, iterator.nextDoc());
+        assertNotNull(vectorValues.vectorValue(iterator.index()));
 
         TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new byte[] { 1, 1, 1 }, 5, null, Integer.MAX_VALUE);
         assertNotNull(topDocs);
@@ -249,7 +252,7 @@ public void testKnnByteVectors() throws Exception {
         // Check that we can't see fieldB
         assertNull(leafReader.getByteVectorValues("fieldB"));
         topDocs = leafReader.searchNearestVectors("fieldB", new byte[] { 1, 1, 1 }, 5, null, Integer.MAX_VALUE);
-        assertEquals(0, topDocs.totalHits.value);
+        assertEquals(0, topDocs.totalHits.value());
         assertEquals(0, topDocs.scoreDocs.length);
 
         TestUtil.checkReader(ir);
@@ -274,11 +277,6 @@ public void testStoredFieldsString() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        {
-            Document d2 = ir.document(0);
-            assertEquals(1, d2.getFields().size());
-            assertEquals("testA", d2.get("fieldA"));
-        }
         {
             Document d2 = ir.storedFields().document(0);
             assertEquals(1, d2.getFields().size());
@@ -306,11 +304,6 @@ public void testStoredFieldsBinary() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        {
-            Document d2 = ir.document(0);
-            assertEquals(1, d2.getFields().size());
-            assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
-        }
         {
             Document d2 = ir.storedFields().document(0);
             assertEquals(1, d2.getFields().size());
@@ -338,11 +331,6 @@ public void testStoredFieldsInt() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        {
-            Document d2 = ir.document(0);
-            assertEquals(1, d2.getFields().size());
-            assertEquals(1, d2.getField("fieldA").numericValue());
-        }
         {
             Document d2 = ir.storedFields().document(0);
             assertEquals(1, d2.getFields().size());
@@ -370,11 +358,6 @@ public void testStoredFieldsLong() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        {
-            Document d2 = ir.document(0);
-            assertEquals(1, d2.getFields().size());
-            assertEquals(1L, d2.getField("fieldA").numericValue());
-        }
         {
             Document d2 = ir.storedFields().document(0);
             assertEquals(1, d2.getFields().size());
@@ -402,11 +385,6 @@ public void testStoredFieldsFloat() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        {
-            Document d2 = ir.document(0);
-            assertEquals(1, d2.getFields().size());
-            assertEquals(1F, d2.getField("fieldA").numericValue());
-        }
         {
             Document d2 = ir.storedFields().document(0);
             assertEquals(1, d2.getFields().size());
@@ -434,11 +412,6 @@ public void testStoredFieldsDouble() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        {
-            Document d2 = ir.document(0);
-            assertEquals(1, d2.getFields().size());
-            assertEquals(1D, d2.getField("fieldA").numericValue());
-        }
         {
             Document d2 = ir.storedFields().document(0);
             assertEquals(1, d2.getFields().size());
@@ -468,7 +441,7 @@ public void testVectors() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Fields vectors = ir.getTermVectors(0);
+        Fields vectors = ir.termVectors().get(0);
         Set<String> seenFields = new HashSet<>();
         for (String field : vectors) {
             seenFields.add(field);
@@ -615,7 +588,6 @@ public void testSortedSetDocValues() throws Exception {
         assertNotNull(dv);
         assertTrue(dv.advanceExact(0));
         assertEquals(0, dv.nextOrd());
-        assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());
         assertEquals(new BytesRef("testA"), dv.lookupOrd(0));
         assertNull(segmentReader.getSortedSetDocValues("fieldB"));
 
@@ -702,11 +674,6 @@ public void testSourceFilteringIntegration() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
 
         // see only one field
-        {
-            Document d2 = ir.document(0);
-            assertEquals(1, d2.getFields().size());
-            assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
-        }
         {
             Document d2 = ir.storedFields().document(0);
             assertEquals(1, d2.getFields().size());
@@ -1201,7 +1168,7 @@ public void testFilterAwayAllVectors() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldB")));
 
         // sees no fields
-        assertNull(ir.getTermVectors(0));
+        assertNull(ir.termVectors().get(0));
 
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
@@ -1229,14 +1196,9 @@ public void testEmpty() throws Exception {
         assertNull(segmentReader.terms("foo"));
 
         // see no vectors
-        assertNull(segmentReader.getTermVectors(0));
         assertNull(segmentReader.termVectors().get(0));
 
         // see no stored fields
-        {
-            Document document = segmentReader.document(0);
-            assertEquals(0, document.getFields().size());
-        }
         {
             Document document = segmentReader.storedFields().document(0);
             assertEquals(0, document.getFields().size());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
index df64c4f87410a..4751f66cf548e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
@@ -193,7 +193,7 @@ protected IndicesAccessControl getIndicesAccessControl() {
             int expectedHitCount = valuesHitCount[i];
             logger.info("Going to verify hit count with query [{}] with expected total hits [{}]", parsedQuery.query(), expectedHitCount);
 
-            Integer totalHits = indexSearcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
+            Integer totalHits = indexSearcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager(indexSearcher.getSlices()));
             assertThat(totalHits, equalTo(expectedHitCount));
             assertThat(wrappedDirectoryReader.numDocs(), equalTo(expectedHitCount));
         }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java
index 394455879bbdf..5b5a895f12ae8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.core.security.authz.permission;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -29,6 +28,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -163,11 +163,11 @@ public void testMonitorEnrichPerVersion() {
         // test monitor_enrich before, after and on monitor enrich version
         String[] privileges = randomBoolean() ? new String[] { "monitor_enrich" } : new String[] { "monitor_enrich", "foo", "bar" };
         String[] before = new RemoteClusterPermissions().addGroup(new RemoteClusterPermissionGroup(privileges, new String[] { "*" }))
-            .privilegeNames("*", TransportVersionUtils.getPreviousVersion(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS));
+            .privilegeNames("*", TransportVersionUtils.getPreviousVersion(ROLE_REMOTE_CLUSTER_PRIVS));
         // empty set since monitor_enrich is not allowed in the before version
         assertThat(Set.of(before), equalTo(Collections.emptySet()));
         String[] on = new RemoteClusterPermissions().addGroup(new RemoteClusterPermissionGroup(privileges, new String[] { "*" }))
-            .privilegeNames("*", TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS);
+            .privilegeNames("*", ROLE_REMOTE_CLUSTER_PRIVS);
         // only monitor_enrich since the other values are not allowed
         assertThat(Set.of(on), equalTo(Set.of("monitor_enrich")));
         String[] after = new RemoteClusterPermissions().addGroup(new RemoteClusterPermissionGroup(privileges, new String[] { "*" }))
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
index 265714ee6ea16..073b3b92a43a5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.core.security.authz.privilege;
 
-import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.delete.TransportDeleteAction;
@@ -17,6 +16,7 @@
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
+import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction;
 
 import java.util.Collection;
@@ -83,7 +83,7 @@ public void testPrivilegesForGetCheckPointAction() {
 
     public void testRelationshipBetweenPrivileges() {
         assertThat(
-            Operations.subsetOf(
+            Automatons.subsetOf(
                 IndexPrivilege.get(Set.of("view_index_metadata")).automaton,
                 IndexPrivilege.get(Set.of("manage")).automaton
             ),
@@ -91,12 +91,12 @@ public void testRelationshipBetweenPrivileges() {
         );
 
         assertThat(
-            Operations.subsetOf(IndexPrivilege.get(Set.of("monitor")).automaton, IndexPrivilege.get(Set.of("manage")).automaton),
+            Automatons.subsetOf(IndexPrivilege.get(Set.of("monitor")).automaton, IndexPrivilege.get(Set.of("manage")).automaton),
             is(true)
         );
 
         assertThat(
-            Operations.subsetOf(
+            Automatons.subsetOf(
                 IndexPrivilege.get(Set.of("create", "create_doc", "index", "delete")).automaton,
                 IndexPrivilege.get(Set.of("write")).automaton
             ),
@@ -104,7 +104,7 @@ public void testRelationshipBetweenPrivileges() {
         );
 
         assertThat(
-            Operations.subsetOf(
+            Automatons.subsetOf(
                 IndexPrivilege.get(Set.of("create_index", "delete_index")).automaton,
                 IndexPrivilege.get(Set.of("manage")).automaton
             ),
@@ -122,7 +122,7 @@ public void testCrossClusterReplicationPrivileges() {
             "indices:admin/seq_no/renew_retention_lease"
         ).forEach(action -> assertThat(crossClusterReplication.predicate.test(action + randomAlphaOfLengthBetween(0, 8)), is(true)));
         assertThat(
-            Operations.subsetOf(crossClusterReplication.automaton, IndexPrivilege.get(Set.of("manage", "read", "monitor")).automaton),
+            Automatons.subsetOf(crossClusterReplication.automaton, IndexPrivilege.get(Set.of("manage", "read", "monitor")).automaton),
             is(true)
         );
 
@@ -139,10 +139,10 @@ public void testCrossClusterReplicationPrivileges() {
             );
 
         assertThat(
-            Operations.subsetOf(crossClusterReplicationInternal.automaton, IndexPrivilege.get(Set.of("manage")).automaton),
+            Automatons.subsetOf(crossClusterReplicationInternal.automaton, IndexPrivilege.get(Set.of("manage")).automaton),
             is(false)
         );
-        assertThat(Operations.subsetOf(crossClusterReplicationInternal.automaton, IndexPrivilege.get(Set.of("all")).automaton), is(true));
+        assertThat(Automatons.subsetOf(crossClusterReplicationInternal.automaton, IndexPrivilege.get(Set.of("all")).automaton), is(true));
     }
 
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java
index 6f3c435eb12f6..a58acf82ea44e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.authz.privilege;
 
-import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction;
 import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction;
@@ -218,13 +218,13 @@ public void testIndexCollapse() throws Exception {
         Set<String> name = Sets.newHashSet(first.name().iterator().next(), second.name().iterator().next());
         IndexPrivilege index = IndexPrivilege.get(name);
 
-        if (Operations.subsetOf(second.getAutomaton(), first.getAutomaton())) {
-            assertTrue(Operations.sameLanguage(index.getAutomaton(), first.getAutomaton()));
-        } else if (Operations.subsetOf(first.getAutomaton(), second.getAutomaton())) {
-            assertTrue(Operations.sameLanguage(index.getAutomaton(), second.getAutomaton()));
+        if (Automatons.subsetOf(second.getAutomaton(), first.getAutomaton())) {
+            assertTrue(AutomatonTestUtil.sameLanguage(index.getAutomaton(), first.getAutomaton()));
+        } else if (Automatons.subsetOf(first.getAutomaton(), second.getAutomaton())) {
+            assertTrue(AutomatonTestUtil.sameLanguage(index.getAutomaton(), second.getAutomaton()));
         } else {
-            assertFalse(Operations.sameLanguage(index.getAutomaton(), first.getAutomaton()));
-            assertFalse(Operations.sameLanguage(index.getAutomaton(), second.getAutomaton()));
+            assertFalse(AutomatonTestUtil.sameLanguage(index.getAutomaton(), first.getAutomaton()));
+            assertFalse(AutomatonTestUtil.sameLanguage(index.getAutomaton(), second.getAutomaton()));
         }
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index 26b306d6f1334..a71ac6a9b51fd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -801,7 +801,7 @@ public void testKibanaSystemRole() {
             assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(mockIndexAbstraction(index)), is(true));
             assertThat(
                 kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(mockIndexAbstraction(index)),
-                is(false)
+                is(true)
             );
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(mockIndexAbstraction(index)), is(false));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(mockIndexAbstraction(index)), is(false));
@@ -949,7 +949,7 @@ public void testKibanaSystemRole() {
             );
         });
 
-        // read-only index for Endpoint and Osquery manager specific action responses
+        // Elastic Defend internal index for response actions results
         Arrays.asList(".logs-endpoint.action.responses-" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> {
             final IndexAbstraction indexAbstraction = mockIndexAbstraction(index);
             assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false));
@@ -959,10 +959,7 @@ public void testKibanaSystemRole() {
                 is(false)
             );
             assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true));
-            assertThat(
-                kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction),
-                is(false)
-            );
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(indexAbstraction), is(true));
@@ -1069,10 +1066,7 @@ public void testKibanaSystemRole() {
                 is(false)
             );
             assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true));
-            assertThat(
-                kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction),
-                is(false)
-            );
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(indexAbstraction), is(true));
@@ -1097,10 +1091,7 @@ public void testKibanaSystemRole() {
                 is(false)
             );
             assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true));
-            assertThat(
-                kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction),
-                is(false)
-            );
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction), is(false));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(indexAbstraction), is(false));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(indexAbstraction), is(true));
@@ -1319,12 +1310,21 @@ public void testKibanaSystemRole() {
 
             final boolean isAlsoAutoCreateIndex = indexName.startsWith(".logs-endpoint.actions-")
                 || indexName.startsWith(".logs-endpoint.action.responses-");
+
+            final boolean isAlsoCreateIndex = indexName.startsWith(".logs-endpoint.actions-")
+                || indexName.startsWith(".logs-endpoint.action.responses-")
+                || indexName.startsWith(".logs-endpoint.diagnostic.collection-")
+                || indexName.startsWith(".logs-endpoint.heartbeat-");
+
             assertThat(
                 kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction),
-                is(false)
+                is(isAlsoCreateIndex)
+            );
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(indexAbstraction), is(isAlsoCreateIndex));
+            assertThat(
+                kibanaRole.indices().allowedIndicesMatcher(CreateDataStreamAction.NAME).test(indexAbstraction),
+                is(isAlsoCreateIndex)
             );
-            assertThat(kibanaRole.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(indexAbstraction), is(isAlsoAutoCreateIndex));
-            assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateDataStreamAction.NAME).test(indexAbstraction), is(false));
             assertThat(
                 kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction),
                 is(isAlsoAutoCreateIndex)
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
index 0b2e48bd20dfe..94f91f427e19a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
@@ -20,7 +20,6 @@
 import java.util.Arrays;
 import java.util.List;
 
-import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
 import static org.elasticsearch.xpack.core.security.support.Automatons.pattern;
 import static org.elasticsearch.xpack.core.security.support.Automatons.patterns;
 import static org.elasticsearch.xpack.core.security.support.Automatons.predicate;
@@ -115,12 +114,12 @@ public void testPatternComplexity() {
     }
 
     private void assertMatch(Automaton automaton, String text) {
-        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_DETERMINIZE_WORK_LIMIT);
+        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton);
         assertTrue(runAutomaton.run(text));
     }
 
     private void assertMismatch(Automaton automaton, String text) {
-        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_DETERMINIZE_WORK_LIMIT);
+        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton);
         assertFalse(runAutomaton.run(text));
     }
 
diff --git a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java
index 9e5f999d1f825..2b9d9b0875220 100644
--- a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java
+++ b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java
@@ -100,8 +100,9 @@ public List<Route> routes() {
             Route.builder(GET, "/_test_cluster/deprecated_settings")
                 .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.current())
                 .build(),
-            // TODO: s/deprecated/deprecatedForRemoval when removing `deprecated` method
-            Route.builder(POST, "/_test_cluster/deprecated_settings").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.current()).build(),
+            Route.builder(POST, "/_test_cluster/deprecated_settings")
+                .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.current())
+                .build(),
             Route.builder(GET, "/_test_cluster/compat_only")
                 .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.minimumSupported())
                 .build(),
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java
index 235209243ee58..f817885ccd9bc 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestUtils;
@@ -24,12 +23,7 @@ public class RestDeprecationInfoAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_migration/deprecations").replaces(GET, "/_xpack/migration/deprecations", RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/_migration/deprecations")
-                .replaces(GET, "/{index}/_xpack/migration/deprecations", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, "/_migration/deprecations"), new Route(GET, "/{index}/_migration/deprecations"));
     }
 
     @Override
diff --git a/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle b/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle
index 61aa2927e46de..6b1c7e42c0fde 100644
--- a/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle
+++ b/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
@@ -26,7 +26,7 @@ restResources {
 }
 
 def supportedVersion = bwcVersion -> {
-  return bwcVersion.onOrAfter("8.10.0");
+  return bwcVersion.onOrAfter("8.10.0") && bwcVersion != VersionProperties.elasticsearchVersion
 }
 
 BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName ->
diff --git a/x-pack/plugin/downsample/qa/rest/build.gradle b/x-pack/plugin/downsample/qa/rest/build.gradle
index ba5ac7b0c7317..5142632a36006 100644
--- a/x-pack/plugin/downsample/qa/rest/build.gradle
+++ b/x-pack/plugin/downsample/qa/rest/build.gradle
@@ -32,6 +32,20 @@ tasks.named('yamlRestTest') {
 tasks.named('yamlRestCompatTest') {
   usesDefaultDistribution()
 }
+tasks.named("yamlRestCompatTestTransform").configure ({ task ->
+  task.skipTest("downsample/10_basic/Downsample index with empty dimension on routing path", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample histogram as label", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample date timestamp field using strict_date_optional_time_nanos format",
+                "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample a downsampled index", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample date_nanos timestamp field using custom format", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample using coarse grained timestamp", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample label with ignore_above", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample object field", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample empty and missing labels", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample index", "Skip until pr/115358 gets backported")
+  task.skipTest("downsample/10_basic/Downsample index with empty dimension", "Skip until pr/115358 gets backported")
+})
 if (BuildParams.inFipsJvm){
   // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
   tasks.named("yamlRestTest").configure{enabled = false }
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java
index 504326f1bd4b1..6794bc47fa3cd 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java
@@ -20,7 +20,7 @@ public class DownsampleRestIT extends ESClientYamlSuiteTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
-        .setting("xpack.license.self_generated.type", "basic")
+        .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "false")
         .build();
 
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java
new file mode 100644
index 0000000000000..8f75e76315844
--- /dev/null
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.downsample;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
+
+public class DownsampleWithBasicRestIT extends ESClientYamlSuiteTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public DownsampleWithBasicRestIT(final ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+}
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
index 0bcd35cc69038..fa3560bec516e 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
@@ -16,6 +16,7 @@ setup:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -106,6 +107,7 @@ setup:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -172,6 +174,7 @@ setup:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -237,6 +240,7 @@ setup:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -318,29 +322,29 @@ setup:
 
   - length: { hits.hits: 4 }
   - match:  { hits.hits.0._source._doc_count: 2 }
-  - match:  { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match:  { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match:  { hits.hits.0._source.metricset: pod }
 
   - match:  { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match:  { hits.hits.0._source.k8s.pod.multi-counter: 0 }
-  - match:  { hits.hits.0._source.k8s.pod.scaled-counter: 0.00 }
-  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.min: 100 }
-  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.max: 102 }
-  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.sum: 607 }
-  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 }
-  - match: { hits.hits.0._source.k8s.pod.scaled-gauge.min: 100.0 }
-  - match: { hits.hits.0._source.k8s.pod.scaled-gauge.max: 101.0 }
-  - match: { hits.hits.0._source.k8s.pod.scaled-gauge.sum: 201.0 }
-  - match: { hits.hits.0._source.k8s.pod.scaled-gauge.value_count: 2 }
-  - match:  { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 }
-  - match:  { hits.hits.0._source.k8s.pod.network.tx.max: 1434577921 }
-  - match:  { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 }
-  - match:  { hits.hits.0._source.k8s.pod.ip: "10.10.55.56" }
-  - match:  { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:43:00.000Z" }
-  - match:  { hits.hits.0._source.k8s.pod.number_of_containers: 1 }
-  - match:  { hits.hits.0._source.k8s.pod.tags: ["backend", "test", "us-west2"] }
-  - match:  { hits.hits.0._source.k8s.pod.values: [1, 1, 2] }
-  - is_false: hits.hits.0._source.k8s.pod.running
+  - match:  { hits.hits.0._source.k8s\.pod\.multi-counter: 0 }
+  - match:  { hits.hits.0._source.k8s\.pod\.scaled-counter: 0.00 }
+  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 100 }
+  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 102 }
+  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 607 }
+  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.value_count: 6 }
+  - match: { hits.hits.0._source.k8s\.pod\.scaled-gauge.min: 100.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.scaled-gauge.max: 101.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.scaled-gauge.sum: 201.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.scaled-gauge.value_count: 2 }
+  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.min: 1434521831 }
+  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.max: 1434577921 }
+  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.value_count: 2 }
+  - match:  { hits.hits.0._source.k8s\.pod\.ip: "10.10.55.56" }
+  - match:  { hits.hits.0._source.k8s\.pod\.created_at: "2021-04-28T19:43:00.000Z" }
+  - match:  { hits.hits.0._source.k8s\.pod\.number_of_containers: 1 }
+  - match:  { hits.hits.0._source.k8s\.pod\.tags: ["backend", "test", "us-west2"] }
+  - match:  { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] }
+  - is_false: hits.hits.0._source.k8s\.pod\.running
 
   # Assert rollup index settings
   - do:
@@ -362,21 +366,21 @@ setup:
   - match: { test-downsample.mappings.properties.@timestamp.type: date }
   - match: { test-downsample.mappings.properties.@timestamp.meta.fixed_interval: 1h }
   - match: { test-downsample.mappings.properties.@timestamp.meta.time_zone: UTC }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.multi-gauge.type: aggregate_metric_double }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.multi-gauge.metrics: [ "min", "max", "sum", "value_count" ] }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.multi-gauge.default_metric: max }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.multi-gauge.time_series_metric: gauge }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.multi-counter.type: long }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.multi-counter.time_series_metric: counter }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.scaled-counter.type: scaled_float }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.scaled-counter.scaling_factor: 100 }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.scaled-counter.time_series_metric: counter }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.scaled-gauge.type: aggregate_metric_double }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.scaled-gauge.metrics: [ "min", "max", "sum", "value_count" ] }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.scaled-gauge.default_metric: max }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.scaled-gauge.time_series_metric: gauge }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.uid.type: keyword }
-  - match: { test-downsample.mappings.properties.k8s.properties.pod.properties.uid.time_series_dimension: true }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.multi-gauge.type: aggregate_metric_double }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.multi-gauge.metrics: [ "min", "max", "sum", "value_count" ] }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.multi-gauge.default_metric: max }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.multi-gauge.time_series_metric: gauge }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.multi-counter.type: long }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.multi-counter.time_series_metric: counter }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-counter.type: scaled_float }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-counter.scaling_factor: 100 }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-counter.time_series_metric: counter }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-gauge.type: aggregate_metric_double }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-gauge.metrics: [ "min", "max", "sum", "value_count" ] }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-gauge.default_metric: max }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.scaled-gauge.time_series_metric: gauge }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.uid.type: keyword }
+  - match: { test-downsample.mappings.properties.k8s\.pod\.uid.time_series_dimension: true }
 
 
   # Assert source index has not been deleted
@@ -763,18 +767,18 @@ setup:
   - match: { test-downsample-2.mappings.properties.@timestamp.type: date }
   - match: { test-downsample-2.mappings.properties.@timestamp.meta.fixed_interval: 2h }
   - match: { test-downsample-2.mappings.properties.@timestamp.meta.time_zone: UTC }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.multi-gauge.type: aggregate_metric_double }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.multi-gauge.metrics: [ "min", "max", "sum", "value_count" ] }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.multi-gauge.default_metric: max }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.multi-gauge.time_series_metric: gauge }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.multi-counter.type: long }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.multi-counter.time_series_metric: counter }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.uid.type: keyword }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.uid.time_series_dimension: true }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.network.properties.tx.type: aggregate_metric_double }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.network.properties.tx.metrics: [ "min", "max", "sum", "value_count" ] }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.network.properties.tx.default_metric: max }
-  - match: { test-downsample-2.mappings.properties.k8s.properties.pod.properties.network.properties.tx.time_series_metric: gauge }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.multi-gauge.type: aggregate_metric_double }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.multi-gauge.metrics: [ "min", "max", "sum", "value_count" ] }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.multi-gauge.default_metric: max }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.multi-gauge.time_series_metric: gauge }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.multi-counter.type: long }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.multi-counter.time_series_metric: counter }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.uid.type: keyword }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.uid.time_series_dimension: true }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.network\.tx.type: aggregate_metric_double }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.network\.tx.metrics: [ "min", "max", "sum", "value_count" ] }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.network\.tx.default_metric: max }
+  - match: { test-downsample-2.mappings.properties.k8s\.pod\.network\.tx.time_series_metric: gauge }
 
   - do:
       search:
@@ -784,29 +788,29 @@ setup:
 
   - length: { hits.hits: 3 }
   - match: { hits.hits.0._source._doc_count: 4 }
-  - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match: { hits.hits.0._source.k8s.pod.multi-counter: 76 }
-  - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 95.0 }
-  - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 110.0 }
-  - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 1209.0 }
-  - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 12 }
-  - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 }
-  - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434595272 }
-  - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 4 }
-  - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.120" }
-  - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:45:00.000Z" }
-  - match: { hits.hits.0._source.k8s.pod.number_of_containers: 1 }
-  - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "test", "us-west1" ] }
-  - match: { hits.hits.0._source.k8s.pod.values: [ 1, 2, 3 ] }
-
-  - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s\.pod\.multi-counter: 76 }
+  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 95.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 110.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 1209.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.value_count: 12 }
+  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.min: 1434521831 }
+  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.max: 1434595272 }
+  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.value_count: 4 }
+  - match: { hits.hits.0._source.k8s\.pod\.ip: "10.10.55.120" }
+  - match: { hits.hits.0._source.k8s\.pod\.created_at: "2021-04-28T19:45:00.000Z" }
+  - match: { hits.hits.0._source.k8s\.pod\.number_of_containers: 1 }
+  - match: { hits.hits.0._source.k8s\.pod\.tags: [ "backend", "test", "us-west1" ] }
+  - match: { hits.hits.0._source.k8s\.pod\.values: [ 1, 2, 3 ] }
+
+  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:00:00.000Z }
   - match: { hits.hits.1._source._doc_count: 2 }
 
-  - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.2._source.metricset: pod }
   - match: { hits.hits.2._source.@timestamp: 2021-04-28T20:00:00.000Z }
   - match: { hits.hits.2._source._doc_count: 2 }
@@ -890,16 +894,16 @@ setup:
   - match: { test-downsample-histogram.mappings.properties.@timestamp.type: date }
   - match: { test-downsample-histogram.mappings.properties.@timestamp.meta.fixed_interval: 1h }
   - match: { test-downsample-histogram.mappings.properties.@timestamp.meta.time_zone: UTC }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.latency.type: histogram }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.latency.time_series_metric: null }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.empty-histogram.type: histogram }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.empty-histogram.time_series_metric: null }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.uid.type: keyword }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.uid.time_series_dimension: true }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.type: aggregate_metric_double }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.metrics: [ "min", "max", "sum", "value_count" ] }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.default_metric: max }
-  - match: { test-downsample-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.time_series_metric: gauge }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.latency.type: histogram }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.latency.time_series_metric: null }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.empty-histogram.type: histogram }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.empty-histogram.time_series_metric: null }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.uid.type: keyword }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.uid.time_series_dimension: true }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.network\.tx.type: aggregate_metric_double }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.network\.tx.metrics: [ "min", "max", "sum", "value_count" ] }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.network\.tx.default_metric: max }
+  - match: { test-downsample-histogram.mappings.properties.k8s\.pod\.network\.tx.time_series_metric: gauge }
 
   - do:
       search:
@@ -910,64 +914,64 @@ setup:
   - length: { hits.hits: 4 }
 
   - match: { hits.hits.0._source._doc_count: 2 }
-  - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - length: { hits.hits.0._source.k8s.pod.latency.counts: 4 }
-  - match: { hits.hits.0._source.k8s.pod.latency.counts.0: 2 }
-  - match: { hits.hits.0._source.k8s.pod.latency.counts.1: 2 }
-  - match: { hits.hits.0._source.k8s.pod.latency.counts.2: 8 }
-  - match: { hits.hits.0._source.k8s.pod.latency.counts.3: 8 }
-  - length: { hits.hits.0._source.k8s.pod.latency.values: 4 }
-  - match: { hits.hits.0._source.k8s.pod.latency.values.0: 1.0 }
-  - match: { hits.hits.0._source.k8s.pod.latency.values.1: 10.0 }
-  - match: { hits.hits.0._source.k8s.pod.latency.values.2: 100.0 }
-  - match: { hits.hits.0._source.k8s.pod.latency.values.3: 1000.0 }
+  - length: { hits.hits.0._source.k8s\.pod\.latency.counts: 4 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.counts.0: 2 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.counts.1: 2 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.counts.2: 8 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.counts.3: 8 }
+  - length: { hits.hits.0._source.k8s\.pod\.latency.values: 4 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.values.0: 1.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.values.1: 10.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.values.2: 100.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.latency.values.3: 1000.0 }
 
   - match: { hits.hits.1._source._doc_count: 1 }
-  - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
-  - length: { hits.hits.1._source.k8s.pod.latency.counts: 4 }
-  - match: { hits.hits.1._source.k8s.pod.latency.counts.0: 4 }
-  - match: { hits.hits.1._source.k8s.pod.latency.counts.1: 5 }
-  - match: { hits.hits.1._source.k8s.pod.latency.counts.2: 4 }
-  - match: { hits.hits.1._source.k8s.pod.latency.counts.3: 13 }
-  - length: { hits.hits.1._source.k8s.pod.latency.values: 4 }
-  - match: { hits.hits.1._source.k8s.pod.latency.values.0: 1.0 }
-  - match: { hits.hits.1._source.k8s.pod.latency.values.1: 10.0 }
-  - match: { hits.hits.1._source.k8s.pod.latency.values.2: 100.0 }
-  - match: { hits.hits.1._source.k8s.pod.latency.values.3: 1000.0 }
+  - length: { hits.hits.1._source.k8s\.pod\.latency.counts: 4 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.counts.0: 4 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.counts.1: 5 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.counts.2: 4 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.counts.3: 13 }
+  - length: { hits.hits.1._source.k8s\.pod\.latency.values: 4 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.values.0: 1.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.values.1: 10.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.values.2: 100.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.latency.values.3: 1000.0 }
 
   - match:  { hits.hits.2._source._doc_count: 2 }
-  - match:  { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match:  { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match:  { hits.hits.2._source.metricset: pod }
   - match:  { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - length: { hits.hits.2._source.k8s.pod.latency.counts: 4 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.counts.0: 8 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.counts.1: 7 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.counts.2: 10 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.counts.3: 12 }
-  - length: { hits.hits.2._source.k8s.pod.latency.values: 4 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.values.0: 1.0 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.values.1: 2.0 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.values.2: 5.0 }
-  - match:  { hits.hits.2._source.k8s.pod.latency.values.3: 10.0 }
+  - length: { hits.hits.2._source.k8s\.pod\.latency.counts: 4 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.counts.0: 8 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.counts.1: 7 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.counts.2: 10 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.counts.3: 12 }
+  - length: { hits.hits.2._source.k8s\.pod\.latency.values: 4 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.values.0: 1.0 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.values.1: 2.0 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.values.2: 5.0 }
+  - match:  { hits.hits.2._source.k8s\.pod\.latency.values.3: 10.0 }
 
   - match:  { hits.hits.3._source._doc_count: 2 }
-  - match:  { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match:  { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match:  { hits.hits.3._source.metricset: pod }
   - match:  { hits.hits.3._source.@timestamp: 2021-04-28T19:00:00.000Z }
-  - length: { hits.hits.3._source.k8s.pod.latency.counts: 4 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.counts.0: 7 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.counts.1: 15 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.counts.2: 10 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.counts.3: 10 }
-  - length: { hits.hits.3._source.k8s.pod.latency.values: 4 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.values.0: 1.0 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.values.1: 2.0 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.values.2: 5.0 }
-  - match:  { hits.hits.3._source.k8s.pod.latency.values.3: 10.0 }
+  - length: { hits.hits.3._source.k8s\.pod\.latency.counts: 4 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.counts.0: 7 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.counts.1: 15 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.counts.2: 10 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.counts.3: 10 }
+  - length: { hits.hits.3._source.k8s\.pod\.latency.values: 4 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.values.0: 1.0 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.values.1: 2.0 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.values.2: 5.0 }
+  - match:  { hits.hits.3._source.k8s\.pod\.latency.values.3: 10.0 }
 
 ---
 "Downsample date_nanos timestamp field using custom format":
@@ -988,6 +992,7 @@ setup:
                 start_time: 2023-02-23T00:00:00Z
                 end_time: 2023-02-24T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date_nanos
@@ -1048,19 +1053,19 @@ setup:
 
   - length: { hits.hits: 2 }
   - match: { hits.hits.0._source._doc_count: 3 }
-  - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2023-02-23T12:00:00.000000000Z }
-  - match: { hits.hits.0._source.k8s.pod.value.min: 8.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.max: 12.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.sum: 30.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.min: 8.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.max: 12.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.sum: 30.0 }
   - match: { hits.hits.1._source._doc_count: 1 }
-  - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: 2023-02-23T13:00:00.000000000Z }
-  - match: { hits.hits.1._source.k8s.pod.value.min: 9.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.max: 9.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.sum: 9.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.min: 9.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.max: 9.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.sum: 9.0 }
 
   - do:
       indices.get_mapping:
@@ -1090,6 +1095,7 @@ setup:
                 start_time: 2023-02-23T00:00:00Z
                 end_time: 2023-02-24T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -1150,19 +1156,19 @@ setup:
 
   - length: { hits.hits: 2 }
   - match: { hits.hits.0._source._doc_count: 3 }
-  - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2023-02-23T12:00:00.000Z }
-  - match: { hits.hits.0._source.k8s.pod.value.min: 8.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.max: 12.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.sum: 30.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.min: 8.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.max: 12.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.sum: 30.0 }
   - match: { hits.hits.1._source._doc_count: 1 }
-  - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: 2023-02-23T13:00:00.000Z }
-  - match: { hits.hits.1._source.k8s.pod.value.min: 9.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.max: 9.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.sum: 9.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.min: 9.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.max: 9.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.sum: 9.0 }
 
   - do:
       indices.get_mapping:
@@ -1192,6 +1198,7 @@ setup:
                 start_time: 2023-02-23T00:00:00Z
                 end_time: 2023-02-27T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -1251,33 +1258,33 @@ setup:
 
   - length: { hits.hits: 4 }
   - match: { hits.hits.0._source._doc_count: 1 }
-  - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2023-02-23 }
-  - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.max: 10.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.sum: 10.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.min: 10.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.max: 10.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.sum: 10.0 }
   - match: { hits.hits.1._source._doc_count: 1 }
-  - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: 2023-02-24 }
-  - match: { hits.hits.1._source.k8s.pod.value.min: 12.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.max: 12.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.sum: 12.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.min: 12.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.max: 12.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.sum: 12.0 }
   - match: { hits.hits.2._source._doc_count: 1 }
-  - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.2._source.metricset: pod }
   - match: { hits.hits.2._source.@timestamp: 2023-02-25 }
-  - match: { hits.hits.2._source.k8s.pod.value.min: 8.0 }
-  - match: { hits.hits.2._source.k8s.pod.value.max: 8.0 }
-  - match: { hits.hits.2._source.k8s.pod.value.sum: 8.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.min: 8.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.max: 8.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.sum: 8.0 }
   - match: { hits.hits.3._source._doc_count: 1 }
-  - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.3._source.metricset: pod }
   - match: { hits.hits.3._source.@timestamp: 2023-02-26 }
-  - match: { hits.hits.3._source.k8s.pod.value.min: 9.0 }
-  - match: { hits.hits.3._source.k8s.pod.value.max: 9.0 }
-  - match: { hits.hits.3._source.k8s.pod.value.sum: 9.0 }
+  - match: { hits.hits.3._source.k8s\.pod\.value.min: 9.0 }
+  - match: { hits.hits.3._source.k8s\.pod\.value.max: 9.0 }
+  - match: { hits.hits.3._source.k8s\.pod\.value.sum: 9.0 }
 
 ---
 "Downsample object field":
@@ -1304,48 +1311,48 @@ setup:
   - length: { hits.hits: 4 }
 
   - match: { hits.hits.0._source._doc_count: 2 }
-  - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" }
-  - match: { hits.hits.0._source.k8s.pod.name: "dog" }
-  - match: { hits.hits.0._source.k8s.pod.value.min: 9.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.max: 16.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.sum: 25.0 }
-  - match: { hits.hits.0._source.k8s.pod.agent.id: "second" }
-  - match: { hits.hits.0._source.k8s.pod.agent.version: "2.1.7" }
+  - match: { hits.hits.0._source.k8s\.pod\.name: "dog" }
+  - match: { hits.hits.0._source.k8s\.pod\.value.min: 9.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.max: 16.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.sum: 25.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.agent\.id: "second" }
+  - match: { hits.hits.0._source.k8s\.pod\.agent\.version: "2.1.7" }
 
   - match: { hits.hits.1._source._doc_count: 2 }
-  - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: "2021-04-28T19:00:00.000Z" }
-  - match: { hits.hits.1._source.k8s.pod.name: "dog" }
-  - match: { hits.hits.1._source.k8s.pod.value.min: 17.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.max: 25.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.sum: 42.0 }
-  - match: { hits.hits.1._source.k8s.pod.agent.id: "second" }
-  - match: { hits.hits.1._source.k8s.pod.agent.version: "2.1.7" }
+  - match: { hits.hits.1._source.k8s\.pod\.name: "dog" }
+  - match: { hits.hits.1._source.k8s\.pod\.value.min: 17.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.max: 25.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.sum: 42.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.agent\.id: "second" }
+  - match: { hits.hits.1._source.k8s\.pod\.agent\.version: "2.1.7" }
 
   - match: { hits.hits.2._source._doc_count: 2 }
-  - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.2._source.metricset: pod }
   - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" }
-  - match: { hits.hits.2._source.k8s.pod.name: "cat" }
-  - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 }
-  - match: { hits.hits.2._source.k8s.pod.value.max: 20.0 }
-  - match: { hits.hits.2._source.k8s.pod.value.sum: 30.0 }
-  - match: { hits.hits.2._source.k8s.pod.agent.id: "first" }
-  - match: { hits.hits.2._source.k8s.pod.agent.version: "2.0.4" }
+  - match: { hits.hits.2._source.k8s\.pod\.name: "cat" }
+  - match: { hits.hits.2._source.k8s\.pod\.value.min: 10.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.max: 20.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.sum: 30.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.agent\.id: "first" }
+  - match: { hits.hits.2._source.k8s\.pod\.agent\.version: "2.0.4" }
 
   - match: { hits.hits.3._source._doc_count: 2 }
-  - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.3._source.metricset: pod }
   - match: { hits.hits.3._source.@timestamp: "2021-04-28T20:00:00.000Z" }
-  - match: { hits.hits.3._source.k8s.pod.name: "cat" }
-  - match: { hits.hits.3._source.k8s.pod.value.min: 12.0 }
-  - match: { hits.hits.3._source.k8s.pod.value.max: 15.0 }
-  - match: { hits.hits.3._source.k8s.pod.value.sum: 27.0 }
-  - match: { hits.hits.3._source.k8s.pod.agent.id: "first" }
-  - match: { hits.hits.3._source.k8s.pod.agent.version: "2.0.4" }
+  - match: { hits.hits.3._source.k8s\.pod\.name: "cat" }
+  - match: { hits.hits.3._source.k8s\.pod\.value.min: 12.0 }
+  - match: { hits.hits.3._source.k8s\.pod\.value.max: 15.0 }
+  - match: { hits.hits.3._source.k8s\.pod\.value.sum: 27.0 }
+  - match: { hits.hits.3._source.k8s\.pod\.agent\.id: "first" }
+  - match: { hits.hits.3._source.k8s\.pod\.agent\.version: "2.0.4" }
 
 ---
 "Downsample empty and missing labels":
@@ -1372,40 +1379,40 @@ setup:
   - length: { hits.hits: 3 }
 
   - match: { hits.hits.2._source._doc_count: 4 }
-  - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.2._source.metricset: pod }
   - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" }
-  - match: { hits.hits.2._source.k8s.pod.name: "cat" }
-  - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 }
-  - match: { hits.hits.2._source.k8s.pod.value.max: 40.0 }
-  - match: { hits.hits.2._source.k8s.pod.value.sum: 100.0 }
-  - match: { hits.hits.2._source.k8s.pod.value.value_count: 4 }
-  - match: { hits.hits.2._source.k8s.pod.label: "abc" }
-  - match: { hits.hits.2._source.k8s.pod.unmapped: "abc" }
+  - match: { hits.hits.2._source.k8s\.pod\.name: "cat" }
+  - match: { hits.hits.2._source.k8s\.pod\.value.min: 10.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.max: 40.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.sum: 100.0 }
+  - match: { hits.hits.2._source.k8s\.pod\.value.value_count: 4 }
+  - match: { hits.hits.2._source.k8s\.pod\.label: "abc" }
+  - match: { hits.hits.2._source.k8s\.pod\.unmapped: "abc" }
 
   - match: { hits.hits.1._source._doc_count: 4 }
-  - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e9597ab }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e9597ab }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: "2021-04-28T18:00:00.000Z" }
-  - match: { hits.hits.1._source.k8s.pod.name: "cat" }
-  - match: { hits.hits.1._source.k8s.pod.value.min: 10.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.max: 40.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.sum: 100.0 }
-  - match: { hits.hits.1._source.k8s.pod.value.value_count: 4 }
-  - match: { hits.hits.1._source.k8s.pod.label: null }
-  - match: { hits.hits.1._source.k8s.pod.unmapped: null }
+  - match: { hits.hits.1._source.k8s\.pod\.name: "cat" }
+  - match: { hits.hits.1._source.k8s\.pod\.value.min: 10.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.max: 40.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.sum: 100.0 }
+  - match: { hits.hits.1._source.k8s\.pod\.value.value_count: 4 }
+  - match: { hits.hits.1._source.k8s\.pod\.label: null }
+  - match: { hits.hits.1._source.k8s\.pod\.unmapped: null }
 
   - match: { hits.hits.0._source._doc_count: 4 }
-  - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" }
-  - match: { hits.hits.0._source.k8s.pod.name: "dog" }
-  - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.max: 40.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.sum: 100.0 }
-  - match: { hits.hits.0._source.k8s.pod.value.value_count: 4 }
-  - match: { hits.hits.0._source.k8s.pod.label: "xyz" }
-  - match: { hits.hits.0._source.k8s.pod.unmapped: "xyz" }
+  - match: { hits.hits.0._source.k8s\.pod\.name: "dog" }
+  - match: { hits.hits.0._source.k8s\.pod\.value.min: 10.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.max: 40.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.sum: 100.0 }
+  - match: { hits.hits.0._source.k8s\.pod\.value.value_count: 4 }
+  - match: { hits.hits.0._source.k8s\.pod\.label: "xyz" }
+  - match: { hits.hits.0._source.k8s\.pod\.unmapped: "xyz" }
 
 
 ---
@@ -1427,6 +1434,7 @@ setup:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -1495,45 +1503,45 @@ setup:
 
   - match: { hits.hits.0._source._doc_count: 2 }
   - match: { hits.hits.0._source.metricset: pod }
-  - match: { hits.hits.0._source.k8s.pod.name: dog }
-  - match: { hits.hits.0._source.k8s.pod.value: 20 }
-  - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
-  - match: { hits.hits.0._source.k8s.pod.label: foo }
+  - match: { hits.hits.0._source.k8s\.pod\.name: dog }
+  - match: { hits.hits.0._source.k8s\.pod\.value: 20 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.k8s\.pod\.label: foo }
   - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
 
   - match: { hits.hits.1._source._doc_count: 2 }
   - match: { hits.hits.1._source.metricset: pod }
-  - match: { hits.hits.1._source.k8s.pod.name: fox }
-  - match: { hits.hits.1._source.k8s.pod.value: 20 }
-  - match: { hits.hits.1._source.k8s.pod.uid: 7393ef8e-489c-11ee-be56-0242ac120002 }
-  - match: { hits.hits.1._source.k8s.pod.label: bar }
+  - match: { hits.hits.1._source.k8s\.pod\.name: fox }
+  - match: { hits.hits.1._source.k8s\.pod\.value: 20 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: 7393ef8e-489c-11ee-be56-0242ac120002 }
+  - match: { hits.hits.1._source.k8s\.pod\.label: bar }
   - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:00:00.000Z }
 
   - match: { hits.hits.2._source._doc_count: 2 }
   - match: { hits.hits.2._source.metricset: pod }
-  - match: { hits.hits.2._source.k8s.pod.name: cat }
-  - match: { hits.hits.2._source.k8s.pod.value: 20 }
-  - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.k8s\.pod\.name: cat }
+  - match: { hits.hits.2._source.k8s\.pod\.value: 20 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   # NOTE: when downsampling a label field we propagate the last (most-recent timestamp-wise) non-null value,
   # ignoring/skipping null values. Here the last document has a value that hits ignore_above ("foofoo") and,
   # as a result, we propagate the value of the previous document ("foo")
-  - match: { hits.hits.2._source.k8s.pod.label: foo }
+  - match: { hits.hits.2._source.k8s\.pod\.label: foo }
   - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
 
   - match: { hits.hits.3._source._doc_count: 2 }
   - match: { hits.hits.3._source.metricset: pod }
-  - match: { hits.hits.3._source.k8s.pod.name: cow }
-  - match: { hits.hits.3._source.k8s.pod.value: 20 }
-  - match: { hits.hits.3._source.k8s.pod.uid: a81ef23a-489c-11ee-be56-0242ac120005 }
-  - match: { hits.hits.3._source.k8s.pod.label: null }
+  - match: { hits.hits.3._source.k8s\.pod\.name: cow }
+  - match: { hits.hits.3._source.k8s\.pod\.value: 20 }
+  - match: { hits.hits.3._source.k8s\.pod\.uid: a81ef23a-489c-11ee-be56-0242ac120005 }
+  - match: { hits.hits.3._source.k8s\.pod\.label: null }
   - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:00:00.000Z }
 
   - do:
       indices.get_mapping:
         index: test-downsample-label-ignore-above
 
-  - match: { test-downsample-label-ignore-above.mappings.properties.k8s.properties.pod.properties.label.type: keyword }
-  - match: { test-downsample-label-ignore-above.mappings.properties.k8s.properties.pod.properties.label.ignore_above: 3 }
+  - match: { test-downsample-label-ignore-above.mappings.properties.k8s\.pod\.label.type: keyword }
+  - match: { test-downsample-label-ignore-above.mappings.properties.k8s\.pod\.label.ignore_above: 3 }
 
 ---
 "Downsample index with empty dimension":
@@ -1555,6 +1563,7 @@ setup:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -1612,11 +1621,11 @@ setup:
 
   - length: { hits.hits: 2 }
   - match: { hits.hits.0._source._doc_count: 3 }
-  - match: { hits.hits.0._source.k8s.pod.name: cat }
-  - match: { hits.hits.0._source.k8s.pod.empty: null }
+  - match: { hits.hits.0._source.k8s\.pod\.name: cat }
+  - match: { hits.hits.0._source.k8s\.pod\.empty: null }
   - match: { hits.hits.1._source._doc_count: 1 }
-  - match: { hits.hits.1._source.k8s.pod.name: cat }
-  - match: { hits.hits.1._source.k8s.pod.empty: "" }
+  - match: { hits.hits.1._source.k8s\.pod\.name: cat }
+  - match: { hits.hits.1._source.k8s\.pod\.empty: "" }
 
 ---
 "Downsample index with empty dimension on routing path":
@@ -1638,6 +1647,7 @@ setup:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
           mappings:
+            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -1695,8 +1705,8 @@ setup:
 
   - length: { hits.hits: 2 }
   - match: { hits.hits.0._source._doc_count: 3 }
-  - match: { hits.hits.0._source.k8s.pod.name: cat }
-  - match: { hits.hits.0._source.k8s.pod.empty: null }
+  - match: { hits.hits.0._source.k8s\.pod\.name: cat }
+  - match: { hits.hits.0._source.k8s\.pod\.empty: null }
   - match: { hits.hits.1._source._doc_count: 1 }
-  - match: { hits.hits.1._source.k8s.pod.name: cat }
-  - match: { hits.hits.1._source.k8s.pod.empty: "" }
+  - match: { hits.hits.1._source.k8s\.pod\.name: cat }
+  - match: { hits.hits.1._source.k8s\.pod\.empty: "" }
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
index d3dcd7ae36f59..65d53d3adabe7 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
@@ -125,7 +125,7 @@ public void testRunner() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = searchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("field1"), is(equalTo("value1")));
@@ -176,7 +176,7 @@ public void testRunner() throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(3)));
@@ -202,7 +202,7 @@ public void testRunnerGeoMatchType() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("location"), is(equalTo("POINT(10.0 10.0)")));
@@ -244,7 +244,7 @@ public void testRunnerGeoMatchType() throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(2)));
@@ -286,7 +286,7 @@ private void testNumberRangeMatchType(String rangeType) throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("range"), is(equalTo(Map.of("lt", 10, "gt", 1))));
@@ -330,7 +330,7 @@ private void testNumberRangeMatchType(String rangeType) throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(2)));
@@ -376,7 +376,7 @@ public void testRunnerRangeTypeWithIpRange() throws Exception {
                 new SearchRequest(sourceIndexName).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
             ),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("subnet"), is(equalTo("10.0.0.0/8")));
@@ -421,7 +421,7 @@ public void testRunnerRangeTypeWithIpRange() throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(2)));
@@ -460,7 +460,7 @@ public void testRunnerMultiSource() throws Exception {
                     new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
                 ),
                 sourceSearchResponse -> {
-                    assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                     Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                     assertNotNull(sourceDocMap);
                     assertThat(sourceDocMap.get("idx"), is(equalTo(targetIdx)));
@@ -522,7 +522,7 @@ public void testRunnerMultiSource() throws Exception {
                 new SearchRequest(".enrich-test1").source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
             ),
             enrichSearchResponse -> {
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(3L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(5)));
@@ -564,7 +564,7 @@ public void testRunnerMultiSourceDocIdCollisions() throws Exception {
                     new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
                 ),
                 sourceSearchResponse -> {
-                    assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                     Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                     assertNotNull(sourceDocMap);
                     assertThat(sourceDocMap.get("idx"), is(equalTo(targetIdx)));
@@ -633,7 +633,7 @@ public void testRunnerMultiSourceDocIdCollisions() throws Exception {
                 new SearchRequest(".enrich-test1").source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
             ),
             enrichSearchResponse -> {
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(3L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(5)));
@@ -688,7 +688,7 @@ public void testRunnerMultiSourceEnrichKeyCollisions() throws Exception {
                     new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
                 ),
                 sourceSearchResponse -> {
-                    assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                     Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                     assertNotNull(sourceDocMap);
                     assertThat(sourceDocMap.get("idx"), is(equalTo(targetIdx)));
@@ -749,7 +749,7 @@ public void testRunnerMultiSourceEnrichKeyCollisions() throws Exception {
                 new SearchRequest(".enrich-test1").source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
             ),
             enrichSearchResponse -> {
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(3L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(5)));
@@ -943,7 +943,7 @@ public void testRunnerObjectSourceMapping() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 Map<?, ?> dataField = ((Map<?, ?>) sourceDocMap.get("data"));
@@ -993,7 +993,7 @@ public void testRunnerObjectSourceMapping() throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(1)));
@@ -1051,7 +1051,7 @@ public void testRunnerExplicitObjectSourceMapping() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 Map<?, ?> dataField = ((Map<?, ?>) sourceDocMap.get("data"));
@@ -1100,7 +1100,7 @@ public void testRunnerExplicitObjectSourceMapping() throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(1)));
@@ -1158,7 +1158,7 @@ public void testRunnerExplicitObjectSourceMappingRangePolicy() throws Exception
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 Map<?, ?> dataField = ((Map<?, ?>) sourceDocMap.get("data"));
@@ -1209,7 +1209,7 @@ public void testRunnerExplicitObjectSourceMappingRangePolicy() throws Exception
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(1)));
@@ -1273,7 +1273,7 @@ public void testRunnerTwoObjectLevelsSourceMapping() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 Map<?, ?> dataField = ((Map<?, ?>) sourceDocMap.get("data"));
@@ -1329,7 +1329,7 @@ public void testRunnerTwoObjectLevelsSourceMapping() throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(1)));
@@ -1395,7 +1395,7 @@ public void testRunnerTwoObjectLevelsSourceMappingRangePolicy() throws Exception
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 Map<?, ?> dataField = ((Map<?, ?>) sourceDocMap.get("data"));
@@ -1450,7 +1450,7 @@ public void testRunnerTwoObjectLevelsSourceMappingRangePolicy() throws Exception
                 new SearchRequest(".enrich-test1").source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))
             ),
             enrichSearchResponse -> {
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(1)));
@@ -1519,7 +1519,7 @@ public void testRunnerTwoObjectLevelsSourceMappingDateRangeWithFormat() throws E
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 Map<?, ?> dataField = ((Map<?, ?>) sourceDocMap.get("data"));
@@ -1580,7 +1580,7 @@ public void testRunnerTwoObjectLevelsSourceMappingDateRangeWithFormat() throws E
                     SearchSourceBuilder.searchSource().query(QueryBuilders.matchQuery("data.fields.period", "2021-08-19T14:00:00Z"))
                 )
             ),
-            enrichSearchResponse -> assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(0L))
+            enrichSearchResponse -> assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(0L))
         );
 
         assertResponse(
@@ -1590,7 +1590,7 @@ public void testRunnerTwoObjectLevelsSourceMappingDateRangeWithFormat() throws E
                 )
             ),
             enrichSearchResponse -> {
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(1)));
@@ -1614,7 +1614,7 @@ public void testRunnerTwoObjectLevelsSourceMappingDateRangeWithFormat() throws E
                     SearchSourceBuilder.searchSource().query(QueryBuilders.matchQuery("data.fields.period", "2021/08/20 at 14:00"))
                 )
             ),
-            enrichSearchResponse -> assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L))
+            enrichSearchResponse -> assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L))
         );
 
         // Validate segments
@@ -1657,7 +1657,7 @@ public void testRunnerDottedKeyNameSourceMapping() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("data.field1"), is(equalTo("value1")));
@@ -1704,7 +1704,7 @@ public void testRunnerDottedKeyNameSourceMapping() throws Exception {
             ),
             enrichSearchResponse -> {
 
-                assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                 assertNotNull(enrichDocument);
                 assertThat(enrichDocument.size(), is(equalTo(2)));
@@ -1736,7 +1736,7 @@ public void testRunnerWithForceMergeRetry() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("field1"), is(equalTo("value1")));
@@ -1868,7 +1868,7 @@ protected void afterRefreshEnrichIndex(ActionListener<Void> listener) {
                 ),
                 enrichSearchResponse -> {
 
-                    assertThat(enrichSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(enrichSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                     Map<String, Object> enrichDocument = enrichSearchResponse.getHits().iterator().next().getSourceAsMap();
                     assertNotNull(enrichDocument);
                     assertThat(enrichDocument.size(), is(equalTo(3)));
@@ -1901,7 +1901,7 @@ public void testRunnerWithEmptySegmentsResponse() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("field1"), is(equalTo("value1")));
@@ -2007,7 +2007,7 @@ public void testRunnerWithShardFailuresInSegmentResponse() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("field1"), is(equalTo("value1")));
@@ -2405,7 +2405,7 @@ public void testRunnerValidatesIndexIntegrity() throws Exception {
         assertResponse(
             client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
             sourceSearchResponse -> {
-                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
                 assertNotNull(sourceDocMap);
                 assertThat(sourceDocMap.get("field1"), is(equalTo("value1")));
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java
index 00f22aca2cb92..8dbc9b0f4f43a 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchActionTests.java
@@ -64,7 +64,7 @@ public void testExecute() throws Exception {
                 assertThat(response.getResponses().length, equalTo(numSearches));
                 for (int i = 0; i < numSearches; i++) {
                     assertThat(response.getResponses()[i].isFailure(), is(false));
-                    assertThat(response.getResponses()[i].getResponse().getHits().getTotalHits().value, equalTo(1L));
+                    assertThat(response.getResponses()[i].getResponse().getHits().getTotalHits().value(), equalTo(1L));
                     assertThat(response.getResponses()[i].getResponse().getHits().getHits()[0].getSourceAsMap().size(), equalTo(1));
                     assertThat(
                         response.getResponses()[i].getResponse().getHits().getHits()[0].getSourceAsMap().get("key1"),
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml
new file mode 100644
index 0000000000000..016d9f10fe77f
--- /dev/null
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml
@@ -0,0 +1,252 @@
+setup:
+  - requires:
+      cluster_features: [ "query_rules.test" ]
+      reason: Introduced in 8.16.0
+
+  - do:
+      query_rules.put_ruleset:
+        ruleset_id: test-ruleset
+        body:
+          rules:
+            - rule_id: rule1
+              type: pinned
+              criteria:
+                - type: exact
+                  metadata: query_string
+                  values: [ search ]
+              actions:
+                ids:
+                  - 'doc1'
+            - rule_id: rule2
+              type: pinned
+              criteria:
+                - type: exact
+                  metadata: query_string
+                  values: [ ui ]
+              actions:
+                docs:
+                  - '_index': 'test-index1'
+                    '_id': 'doc2'
+            - rule_id: rule3
+              type: pinned
+              criteria:
+                - type: contains
+                  metadata: query_string
+                  values: [ kibana, logstash ]
+              actions:
+                ids:
+                  - 'doc2'
+                  - 'doc3'
+            - rule_id: rule4
+              type: pinned
+              criteria:
+                - type: exact
+                  metadata: query_string
+                  values: [ ops ]
+              actions:
+                ids:
+                  - 'doc7'
+            - rule_id: rule5
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: query_string
+                  values: [ search ]
+              actions:
+                ids:
+                  - 'doc8'
+
+---
+teardown:
+  - do:
+      query_rules.delete_ruleset:
+        ruleset_id: test-ruleset
+        ignore: 404
+
+  - do:
+      query_rules.delete_ruleset:
+        ruleset_id: combined-ruleset
+        ignore: 404
+
+  - do:
+      query_rules.delete_ruleset:
+        ruleset_id: double-jeopardy-ruleset
+        ignore: 404
+
+---
+"Test query rules, specifying a ruleset that does not exist":
+  - do:
+      catch: /resource_not_found_exception/
+      query_rules.test:
+        ruleset_id: nonexistent-ruleset
+        body:
+          match_criteria:
+            foo: bar
+
+
+---
+"Test query rules with an empty body":
+  - do:
+      catch: bad_request
+      query_rules.test:
+        ruleset_id: nonexistent-ruleset
+        body: { }
+
+---
+"Test query rules with an ID match":
+
+  - do:
+      query_rules.test:
+        ruleset_id: test-ruleset
+        body:
+          match_criteria:
+            query_string: search
+
+  - match: { total_matched_rules: 2 }
+  - match: { matched_rules.0.rule_id: 'rule1' }
+  - match: { matched_rules.1.rule_id: 'rule5' }
+
+---
+"As a user, test query rules with an ID match":
+  - skip:
+      features: headers
+
+  - do:
+      catch: forbidden
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
+      query_rules.test:
+        ruleset_id: test-ruleset
+        body:
+          match_criteria:
+            query_string: search
+
+---
+"Test query rules with a doc match":
+
+  - do:
+      query_rules.test:
+        ruleset_id: test-ruleset
+        body:
+          match_criteria:
+            query_string: ui
+
+  - match: { total_matched_rules: 1 }
+  - match: { matched_rules.0.rule_id: 'rule2' }
+
+---
+"As a user, test query rules with a doc match":
+  - skip:
+      features: headers
+
+  - do:
+      catch: forbidden
+      headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
+      query_rules.test:
+        ruleset_id: test-ruleset
+        body:
+          match_criteria:
+            query_string: ui
+
+---
+"Test query rules with no matching rules":
+
+  - do:
+      query_rules.test:
+        ruleset_id: test-ruleset
+        body:
+          match_criteria:
+            query_string: no-match
+
+  - match: { total_matched_rules: 0 }
+
+---
+"Test rules where the same ID is both pinned and excluded":
+  - do:
+      query_rules.put_ruleset:
+        ruleset_id: double-jeopardy-ruleset
+        body:
+          rules:
+            - rule_id: rule1
+              type: pinned
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                ids:
+                  - 'doc8'
+            - rule_id: rule2
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                ids:
+                  - 'doc8'
+
+  - do:
+      query_rules.test:
+        ruleset_id: double-jeopardy-ruleset
+        body:
+          match_criteria:
+            foo: bar
+
+  - match: { total_matched_rules: 2 }
+  - match: { matched_rules.0.rule_id: 'rule1' }
+  - match: { matched_rules.1.rule_id: 'rule2' }
+
+---
+"Perform a rule query over a ruleset with combined numeric and text rule matching":
+
+  - do:
+      query_rules.put_ruleset:
+        ruleset_id: combined-ruleset
+        body:
+          rules:
+            - rule_id: rule1
+              type: pinned
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                ids:
+                  - 'doc1'
+            - rule_id: rule2
+              type: pinned
+              criteria:
+                - type: lte
+                  metadata: foo
+                  values: [ 100 ]
+              actions:
+                ids:
+                  - 'doc2'
+  - do:
+      query_rules.test:
+        ruleset_id: combined-ruleset
+        body:
+          match_criteria:
+            foo: 100
+
+  - match: { total_matched_rules: 1 }
+  - match: { matched_rules.0.rule_id: 'rule2' }
+
+  - do:
+      query_rules.test:
+        ruleset_id: combined-ruleset
+        body:
+          match_criteria:
+            foo: bar
+
+  - match: { total_matched_rules: 1 }
+  - match: { matched_rules.0.rule_id: 'rule1' }
+
+  - do:
+      query_rules.test:
+        ruleset_id: combined-ruleset
+        body:
+          match_criteria:
+            foo: baz
+
+  - match: { total_matched_rules: 0 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java
index bdd4cae3dda81..d5aef3b8808e8 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java
@@ -165,6 +165,8 @@
 import org.elasticsearch.xpack.application.rules.action.RestListQueryRulesetsAction;
 import org.elasticsearch.xpack.application.rules.action.RestPutQueryRuleAction;
 import org.elasticsearch.xpack.application.rules.action.RestPutQueryRulesetAction;
+import org.elasticsearch.xpack.application.rules.action.RestTestQueryRulesetAction;
+import org.elasticsearch.xpack.application.rules.action.TestQueryRulesetAction;
 import org.elasticsearch.xpack.application.rules.action.TransportDeleteQueryRuleAction;
 import org.elasticsearch.xpack.application.rules.action.TransportDeleteQueryRulesetAction;
 import org.elasticsearch.xpack.application.rules.action.TransportGetQueryRuleAction;
@@ -172,6 +174,7 @@
 import org.elasticsearch.xpack.application.rules.action.TransportListQueryRulesetsAction;
 import org.elasticsearch.xpack.application.rules.action.TransportPutQueryRuleAction;
 import org.elasticsearch.xpack.application.rules.action.TransportPutQueryRulesetAction;
+import org.elasticsearch.xpack.application.rules.action.TransportTestQueryRulesetAction;
 import org.elasticsearch.xpack.application.search.SearchApplicationIndexService;
 import org.elasticsearch.xpack.application.search.action.DeleteSearchApplicationAction;
 import org.elasticsearch.xpack.application.search.action.GetSearchApplicationAction;
@@ -266,6 +269,7 @@ protected XPackLicenseState getLicenseState() {
                 new ActionHandler<>(DeleteQueryRuleAction.INSTANCE, TransportDeleteQueryRuleAction.class),
                 new ActionHandler<>(GetQueryRuleAction.INSTANCE, TransportGetQueryRuleAction.class),
                 new ActionHandler<>(PutQueryRuleAction.INSTANCE, TransportPutQueryRuleAction.class),
+                new ActionHandler<>(TestQueryRulesetAction.INSTANCE, TransportTestQueryRulesetAction.class),
 
                 usageAction,
                 infoAction
@@ -373,7 +377,8 @@ public List<RestHandler> getRestHandlers(
                 new RestPutQueryRulesetAction(getLicenseState()),
                 new RestDeleteQueryRuleAction(getLicenseState()),
                 new RestGetQueryRuleAction(getLicenseState()),
-                new RestPutQueryRuleAction(getLicenseState())
+                new RestPutQueryRuleAction(getLicenseState()),
+                new RestTestQueryRulesetAction(getLicenseState())
             )
         );
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java
index 81e072479d402..174bcbe886dfb 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java
@@ -14,8 +14,17 @@
 import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 
 import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.application.rules.action.TestQueryRulesetAction.QUERY_RULES_TEST_API;
 
 public class EnterpriseSearchFeatures implements FeatureSpecification {
+
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(QUERY_RULES_TEST_API);
+    }
+
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
         return Map.of(
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
index cd98b43adc159..5e1fde0dfb942 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
@@ -1073,7 +1073,7 @@ private static ConnectorIndexService.ConnectorResult mapSearchResponseToConnecto
         final List<ConnectorSearchResult> connectorResults = Arrays.stream(response.getHits().getHits())
             .map(ConnectorIndexService::hitToConnector)
             .toList();
-        return new ConnectorIndexService.ConnectorResult(connectorResults, (int) response.getHits().getTotalHits().value);
+        return new ConnectorIndexService.ConnectorResult(connectorResults, (int) response.getHits().getTotalHits().value());
     }
 
     private static ConnectorSearchResult hitToConnector(SearchHit searchHit) {
@@ -1115,7 +1115,7 @@ private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, A
             client.search(searchRequest, new ActionListener<>() {
                 @Override
                 public void onResponse(SearchResponse searchResponse) {
-                    boolean indexNameIsInUse = searchResponse.getHits().getTotalHits().value > 0L;
+                    boolean indexNameIsInUse = searchResponse.getHits().getTotalHits().value() > 0L;
                     listener.onResponse(indexNameIsInUse);
                 }
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
index 9ef895a3a5786..ce6f7f0dbf2b2 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
@@ -417,7 +417,7 @@ private ConnectorSyncJobsResult mapSearchResponseToConnectorSyncJobsList(SearchR
             .map(ConnectorSyncJobIndexService::hitToConnectorSyncJob)
             .toList();
 
-        return new ConnectorSyncJobsResult(connectorSyncJobs, (int) searchResponse.getHits().getTotalHits().value);
+        return new ConnectorSyncJobsResult(connectorSyncJobs, (int) searchResponse.getHits().getTotalHits().value());
     }
 
     private static ConnectorSyncJobSearchResult hitToConnectorSyncJob(SearchHit searchHit) {
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java
index 0ecb35531ac09..e1734268fbf46 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java
@@ -130,7 +130,7 @@ public QueryRule(StreamInput in) throws IOException {
         this.criteria = in.readCollectionAsList(QueryRuleCriteria::new);
         this.actions = in.readGenericMap();
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.QUERY_RULE_CRUD_API_PUT)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.priority = in.readOptionalVInt();
         } else {
             this.priority = null;
@@ -175,7 +175,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(type.toString());
         out.writeCollection(criteria);
         out.writeGenericMap(actions);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.QUERY_RULE_CRUD_API_PUT)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeOptionalVInt(priority);
         }
     }
@@ -331,12 +331,8 @@ public AppliedQueryRules applyRule(AppliedQueryRules appliedRules, Map<String, O
         return new AppliedQueryRules(pinnedDocs, excludedDocs);
     }
 
-    @SuppressWarnings("unchecked")
-    private List<SpecifiedDocument> identifyMatchingDocs(Map<String, Object> matchCriteria) {
-        List<SpecifiedDocument> matchingDocs = new ArrayList<>();
+    public boolean isRuleMatch(Map<String, Object> matchCriteria) {
         Boolean isRuleMatch = null;
-
-        // All specified criteria in a rule must match for the rule to be applied
         for (QueryRuleCriteria criterion : criteria) {
             for (String match : matchCriteria.keySet()) {
                 final Object matchValue = matchCriteria.get(match);
@@ -349,8 +345,13 @@ private List<SpecifiedDocument> identifyMatchingDocs(Map<String, Object> matchCr
                 }
             }
         }
+        return isRuleMatch != null && isRuleMatch;
+    }
 
-        if (isRuleMatch != null && isRuleMatch) {
+    @SuppressWarnings("unchecked")
+    private List<SpecifiedDocument> identifyMatchingDocs(Map<String, Object> matchCriteria) {
+        List<SpecifiedDocument> matchingDocs = new ArrayList<>();
+        if (isRuleMatch(matchCriteria)) {
             if (actions.containsKey(IDS_FIELD.getPreferredName())) {
                 matchingDocs.addAll(
                     ((List<String>) actions.get(IDS_FIELD.getPreferredName())).stream().map(id -> new SpecifiedDocument(null, id)).toList()
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java
index 2eec155ae8ea2..8bf4bbd5716b7 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java
@@ -436,7 +436,7 @@ private static QueryRulesetResult mapSearchResponseToQueryRulesetList(SearchResp
         final List<QueryRulesetListItem> rulesetResults = Arrays.stream(response.getHits().getHits())
             .map(QueryRulesIndexService::hitToQueryRulesetListItem)
             .toList();
-        return new QueryRulesetResult(rulesetResults, (int) response.getHits().getTotalHits().value);
+        return new QueryRulesetResult(rulesetResults, (int) response.getHits().getTotalHits().value());
     }
 
     private static QueryRulesetListItem hitToQueryRulesetListItem(SearchHit searchHit) {
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java
index a3703a5005979..3e573929731fb 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java
@@ -88,7 +88,7 @@ public RuleQueryBuilder(StreamInput in) throws IOException {
         super(in);
         organicQuery = in.readNamedWriteable(QueryBuilder.class);
         matchCriteria = in.readGenericMap();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.RULE_QUERY_RENAME)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rulesetIds = in.readStringCollectionAsList();
         } else {
             rulesetIds = List.of(in.readString());
@@ -144,7 +144,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeNamedWriteable(organicQuery);
         out.writeGenericMap(matchCriteria);
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.RULE_QUERY_RENAME)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeStringCollection(rulesetIds);
         } else {
             out.writeString(rulesetIds.get(0));
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java
index f7e6f166cf53f..1d5ba878264f7 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java
@@ -31,7 +31,8 @@
 
 public class GetQueryRulesetAction {
 
-    public static final String NAME = "cluster:admin/xpack/query_rules/get";
+    public static final ActionType<GetQueryRulesetAction.Response> TYPE = new ActionType<>("cluster:admin/xpack/query_rules/get");
+    public static final String NAME = TYPE.name();
     public static final ActionType<GetQueryRulesetAction.Response> INSTANCE = new ActionType<>(NAME);
 
     private GetQueryRulesetAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetAction.java
new file mode 100644
index 0000000000000..b6e02b3c37262
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetAction.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.rules.action;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.application.EnterpriseSearch;
+import org.elasticsearch.xpack.application.EnterpriseSearchBaseRestHandler;
+import org.elasticsearch.xpack.application.utils.LicenseUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestTestQueryRulesetAction extends EnterpriseSearchBaseRestHandler {
+    public RestTestQueryRulesetAction(XPackLicenseState licenseState) {
+        super(licenseState, LicenseUtils.Product.QUERY_RULES);
+    }
+
+    @Override
+    public String getName() {
+        return "query_ruleset_test_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(POST, "/" + EnterpriseSearch.QUERY_RULES_API_ENDPOINT + "/{ruleset_id}" + "/_test"));
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        final String rulesetId = restRequest.param("ruleset_id");
+        TestQueryRulesetAction.Request request = null;
+        if (restRequest.hasContent()) {
+            try (var parser = restRequest.contentParser()) {
+                request = TestQueryRulesetAction.Request.parse(parser, rulesetId);
+            }
+        }
+        final TestQueryRulesetAction.Request finalRequest = request;
+        return channel -> client.execute(TestQueryRulesetAction.INSTANCE, finalRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java
new file mode 100644
index 0000000000000..28f4a3b38dd59
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.rules.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class TestQueryRulesetAction {
+
+    public static final NodeFeature QUERY_RULES_TEST_API = new NodeFeature("query_rules.test");
+
+    // TODO - We'd like to transition this to require less stringent permissions
+    public static final ActionType<TestQueryRulesetAction.Response> TYPE = new ActionType<>("cluster:admin/xpack/query_rules/test");
+
+    public static final String NAME = TYPE.name();
+    public static final ActionType<TestQueryRulesetAction.Response> INSTANCE = new ActionType<>(NAME);
+
+    private TestQueryRulesetAction() {/* no instances */}
+
+    public static class Request extends ActionRequest implements ToXContentObject, IndicesRequest {
+        private final String rulesetId;
+        private final Map<String, Object> matchCriteria;
+
+        private static final ParseField RULESET_ID_FIELD = new ParseField("ruleset_id");
+        private static final ParseField MATCH_CRITERIA_FIELD = new ParseField("match_criteria");
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.rulesetId = in.readString();
+            this.matchCriteria = in.readGenericMap();
+        }
+
+        public Request(String rulesetId, Map<String, Object> matchCriteria) {
+            this.rulesetId = rulesetId;
+            this.matchCriteria = matchCriteria;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+
+            if (Strings.isNullOrEmpty(rulesetId)) {
+                validationException = addValidationError("ruleset_id missing", validationException);
+            }
+
+            return validationException;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(rulesetId);
+            out.writeGenericMap(matchCriteria);
+        }
+
+        public String rulesetId() {
+            return rulesetId;
+        }
+
+        public Map<String, Object> matchCriteria() {
+            return matchCriteria;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Objects.equals(rulesetId, request.rulesetId) && Objects.equals(matchCriteria, request.matchCriteria);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(rulesetId, matchCriteria);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(RULESET_ID_FIELD.getPreferredName(), rulesetId);
+            builder.startObject(MATCH_CRITERIA_FIELD.getPreferredName());
+            builder.mapContents(matchCriteria);
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        }
+
+        private static final ConstructingObjectParser<Request, String> PARSER = new ConstructingObjectParser<>(
+            "test_query_ruleset_request",
+            false,
+            (p, name) -> {
+                @SuppressWarnings("unchecked")
+                Map<String, Object> matchCriteria = (Map<String, Object>) p[0];
+                return new Request(name, matchCriteria);
+            }
+
+        );
+        static {
+            PARSER.declareObject(constructorArg(), (p, c) -> p.map(), MATCH_CRITERIA_FIELD);
+            PARSER.declareString(optionalConstructorArg(), RULESET_ID_FIELD); // Required for parsing
+        }
+
+        public static Request parse(XContentParser parser, String name) {
+            return PARSER.apply(parser, name);
+        }
+
+        @Override
+        public String[] indices() {
+            return new String[] { QueryRulesIndexService.QUERY_RULES_ALIAS_NAME };
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return IndicesOptions.lenientExpandHidden();
+        }
+
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private final int totalMatchedRules;
+        private final List<MatchedRule> matchedRules;
+
+        private static final ParseField TOTAL_MATCHED_RULES_FIELD = new ParseField("total_matched_rules");
+        private static final ParseField MATCHED_RULES_FIELD = new ParseField("matched_rules");
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            this.totalMatchedRules = in.readVInt();
+            this.matchedRules = in.readCollectionAsList(MatchedRule::new);
+        }
+
+        public Response(int totalMatchedRules, List<MatchedRule> matchedRules) {
+            this.totalMatchedRules = totalMatchedRules;
+            this.matchedRules = matchedRules;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVInt(totalMatchedRules);
+            out.writeCollection(matchedRules, (stream, matchedRule) -> matchedRule.writeTo(stream));
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(TOTAL_MATCHED_RULES_FIELD.getPreferredName(), totalMatchedRules);
+            builder.startArray(MATCHED_RULES_FIELD.getPreferredName());
+            for (MatchedRule matchedRule : matchedRules) {
+                builder.startObject();
+                builder.field("ruleset_id", matchedRule.rulesetId());
+                builder.field("rule_id", matchedRule.ruleId());
+                builder.endObject();
+            }
+            builder.endArray();
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response response = (Response) o;
+            return Objects.equals(totalMatchedRules, response.totalMatchedRules) && Objects.equals(matchedRules, response.matchedRules);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(totalMatchedRules, matchedRules);
+        }
+    }
+
+    public record MatchedRule(String rulesetId, String ruleId) {
+        public MatchedRule(StreamInput in) throws IOException {
+            this(in.readString(), in.readString());
+        }
+
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(rulesetId);
+            out.writeString(ruleId);
+        }
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportTestQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportTestQueryRulesetAction.java
new file mode 100644
index 0000000000000..115cdc516831a
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportTestQueryRulesetAction.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.rules.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.application.rules.QueryRule;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+
+public class TransportTestQueryRulesetAction extends HandledTransportAction<
+    TestQueryRulesetAction.Request,
+    TestQueryRulesetAction.Response> {
+
+    private final Client client;
+
+    @Inject
+    public TransportTestQueryRulesetAction(TransportService transportService, ActionFilters actionFilters, Client client) {
+        super(
+            TestQueryRulesetAction.NAME,
+            transportService,
+            actionFilters,
+            TestQueryRulesetAction.Request::new,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.client = client;
+    }
+
+    @Override
+    protected void doExecute(Task task, TestQueryRulesetAction.Request request, ActionListener<TestQueryRulesetAction.Response> listener) {
+        GetQueryRulesetAction.Request getQueryRulesetRequest = new GetQueryRulesetAction.Request(request.rulesetId());
+        executeAsyncWithOrigin(
+            client,
+            ENT_SEARCH_ORIGIN,
+            GetQueryRulesetAction.TYPE,
+            getQueryRulesetRequest,
+            ActionListener.wrap(getQueryRulesetResponse -> {
+                List<TestQueryRulesetAction.MatchedRule> matchedRules = new ArrayList<>();
+                for (QueryRule rule : getQueryRulesetResponse.queryRuleset().rules()) {
+                    if (rule.isRuleMatch(request.matchCriteria())) {
+                        matchedRules.add(new TestQueryRulesetAction.MatchedRule(request.rulesetId(), rule.id()));
+                    }
+                }
+                listener.onResponse(new TestQueryRulesetAction.Response(matchedRules.size(), matchedRules));
+            }, listener::onFailure)
+        );
+    }
+
+}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java
index 9e8a8f750b764..30d533aeb9ae5 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java
@@ -416,7 +416,7 @@ private static SearchApplicationResult mapSearchResponse(SearchResponse response
         final List<SearchApplicationListItem> apps = Arrays.stream(response.getHits().getHits())
             .map(SearchApplicationIndexService::hitToSearchApplicationListItem)
             .toList();
-        return new SearchApplicationResult(apps, (int) response.getHits().getTotalHits().value);
+        return new SearchApplicationResult(apps, (int) response.getHits().getTotalHits().value());
     }
 
     private static SearchApplicationListItem hitToSearchApplicationListItem(SearchHit searchHit) {
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java
index 06adb29e32691..190b3c3e53169 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java
@@ -115,4 +115,8 @@ public static QueryRuleset randomQueryRuleset() {
         return new QueryRuleset(id, rules);
     }
 
+    public static Map<String, Object> randomMatchCriteria() {
+        return randomMap(1, 3, () -> Tuple.tuple(randomIdentifier(), randomAlphaOfLengthBetween(0, 10)));
+    }
+
 }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java
index b60747ce47321..1f1449d46264b 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java
@@ -52,8 +52,6 @@ protected GetQueryRuleAction.Response mutateInstanceForVersion(GetQueryRuleActio
 
     @Override
     protected List<TransportVersion> bwcVersions() {
-        return getAllBWCVersions().stream()
-            .filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_CRUD_API_GET_DELETE))
-            .collect(Collectors.toList());
+        return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.V_8_15_0)).collect(Collectors.toList());
     }
 }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java
index ca58e42ba6248..8bfbcd5ec471b 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java
@@ -61,7 +61,7 @@ protected GetQueryRulesetAction.Response mutateInstanceForVersion(GetQueryRulese
                 rules.add(new QueryRule(rule.id(), rule.type(), newCriteria, rule.actions(), null));
             }
             return new GetQueryRulesetAction.Response(new QueryRuleset(instance.queryRuleset().id(), rules));
-        } else if (version.before(TransportVersions.QUERY_RULE_CRUD_API_PUT)) {
+        } else if (version.before(TransportVersions.V_8_15_0)) {
             List<QueryRule> rules = new ArrayList<>();
             for (QueryRule rule : instance.queryRuleset().rules()) {
                 rules.add(new QueryRule(rule.id(), rule.type(), rule.criteria(), rule.actions(), null));
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java
index 5b42a899feac8..b60b8718e8549 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java
@@ -55,8 +55,6 @@ protected PutQueryRuleAction.Request mutateInstanceForVersion(PutQueryRuleAction
 
     @Override
     protected List<TransportVersion> bwcVersions() {
-        return getAllBWCVersions().stream()
-            .filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_CRUD_API_PUT))
-            .collect(Collectors.toList());
+        return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.V_8_15_0)).collect(Collectors.toList());
     }
 }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java
index f9b47f5bb2cd2..3b6195cc8f495 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java
@@ -63,7 +63,7 @@ protected PutQueryRulesetAction.Request mutateInstanceForVersion(PutQueryRuleset
                 rules.add(new QueryRule(rule.id(), rule.type(), newCriteria, rule.actions(), null));
             }
             return new PutQueryRulesetAction.Request(new QueryRuleset(instance.queryRuleset().id(), rules));
-        } else if (version.before(TransportVersions.QUERY_RULE_CRUD_API_PUT)) {
+        } else if (version.before(TransportVersions.V_8_15_0)) {
             List<QueryRule> rules = new ArrayList<>();
             for (QueryRule rule : instance.queryRuleset().rules()) {
                 rules.add(new QueryRule(rule.id(), rule.type(), rule.criteria(), rule.actions(), null));
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetActionTests.java
new file mode 100644
index 0000000000000..dc2869e3ff0be
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetActionTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.rules.action;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.application.AbstractRestEnterpriseSearchActionTests;
+import org.elasticsearch.xpack.application.EnterpriseSearchBaseRestHandler;
+import org.elasticsearch.xpack.application.utils.LicenseUtils;
+
+import java.util.Map;
+
+public class RestTestQueryRulesetActionTests extends AbstractRestEnterpriseSearchActionTests {
+    public void testWithNonCompliantLicense() throws Exception {
+        checkLicenseForRequest(
+            new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withMethod(RestRequest.Method.POST)
+                .withParams(Map.of("ruleset_id", "ruleset-id"))
+                .withContent(new BytesArray("""
+                    {
+                      "match_criteria": {
+                        "foo": "bar"
+                      }
+                    }
+                    """), XContentType.JSON)
+                .build(),
+            LicenseUtils.Product.QUERY_RULES
+        );
+    }
+
+    public void testInvalidRequestWithNonCompliantLicense() throws Exception {
+        checkLicenseForRequest(
+            new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withMethod(RestRequest.Method.POST)
+                .withParams(Map.of("invalid_param_name", "invalid_value"))
+                .withContent(new BytesArray("{}"), XContentType.JSON)
+                .build(),
+            LicenseUtils.Product.QUERY_RULES
+        );
+    }
+
+    @Override
+    protected EnterpriseSearchBaseRestHandler getRestAction(XPackLicenseState licenseState) {
+        return new RestTestQueryRulesetAction(licenseState);
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java
new file mode 100644
index 0000000000000..7041de1106b50
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.rules.action;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.test.BWCVersions.getAllBWCVersions;
+
+public class TestQueryRulesetActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase<TestQueryRulesetAction.Request> {
+
+    private final String RULESET_NAME = "my-ruleset";
+
+    @Override
+    protected Writeable.Reader<TestQueryRulesetAction.Request> instanceReader() {
+        return TestQueryRulesetAction.Request::new;
+    }
+
+    @Override
+    protected TestQueryRulesetAction.Request createTestInstance() {
+        return new TestQueryRulesetAction.Request(RULESET_NAME, EnterpriseSearchModuleTestUtils.randomMatchCriteria());
+    }
+
+    @Override
+    protected TestQueryRulesetAction.Request mutateInstance(TestQueryRulesetAction.Request instance) {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+
+    @Override
+    protected TestQueryRulesetAction.Request doParseInstance(XContentParser parser) throws IOException {
+        return TestQueryRulesetAction.Request.parse(parser, RULESET_NAME);
+    }
+
+    @Override
+    protected TestQueryRulesetAction.Request mutateInstanceForVersion(TestQueryRulesetAction.Request instance, TransportVersion version) {
+        return instance;
+    }
+
+    @Override
+    protected List<TransportVersion> bwcVersions() {
+        return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_TEST_API)).collect(Collectors.toList());
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java
new file mode 100644
index 0000000000000..a6562fb7b52af
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.rules.action;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.BWCVersions.getAllBWCVersions;
+
+public class TestQueryRulesetActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase<
+    TestQueryRulesetAction.Response> {
+
+    @Override
+    protected Writeable.Reader<TestQueryRulesetAction.Response> instanceReader() {
+        return TestQueryRulesetAction.Response::new;
+    }
+
+    @Override
+    protected TestQueryRulesetAction.Response mutateInstance(TestQueryRulesetAction.Response instance) {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+
+    @Override
+    protected TestQueryRulesetAction.Response createTestInstance() {
+        int totalMatchedRules = randomIntBetween(0, 10);
+        List<TestQueryRulesetAction.MatchedRule> matchedRules = IntStream.range(0, totalMatchedRules)
+            .mapToObj(i -> new TestQueryRulesetAction.MatchedRule(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)))
+            .toList();
+        return new TestQueryRulesetAction.Response(totalMatchedRules, matchedRules);
+    }
+
+    @Override
+    protected TestQueryRulesetAction.Response mutateInstanceForVersion(TestQueryRulesetAction.Response instance, TransportVersion version) {
+        return instance;
+    }
+
+    @Override
+    protected List<TransportVersion> bwcVersions() {
+        return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_TEST_API)).collect(Collectors.toList());
+    }
+}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
index be1d4c0871ca7..2b7b8b074fa71 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
@@ -582,8 +582,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.startObject(Fields.HITS);
             if (totalHits != null) {
                 builder.startObject(Fields.TOTAL);
-                builder.field("value", totalHits.value);
-                builder.field("relation", totalHits.relation == TotalHits.Relation.EQUAL_TO ? "eq" : "gte");
+                builder.field("value", totalHits.value());
+                builder.field("relation", totalHits.relation() == TotalHits.Relation.EQUAL_TO ? "eq" : "gte");
                 builder.endObject();
             }
             if (events != null) {
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
index 51f92bcda7da4..c0141da2432ce 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
@@ -59,7 +59,6 @@
 import static org.elasticsearch.action.ActionListener.wrap;
 import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName;
 import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
-import static org.elasticsearch.xpack.ql.plugin.TransportActionUtils.executeRequestWithRetryAttempt;
 
 public final class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRequest, EqlSearchResponse>
     implements
@@ -236,22 +235,11 @@ public static void operation(
                 new TaskId(nodeId, task.getId()),
                 task
             );
-            executeRequestWithRetryAttempt(
-                clusterService,
-                listener::onFailure,
-                onFailure -> planExecutor.eql(
-                    cfg,
-                    request.query(),
-                    params,
-                    wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), onFailure)
-                ),
-                node -> transportService.sendRequest(
-                    node,
-                    EqlSearchAction.NAME,
-                    request,
-                    new ActionListenerResponseHandler<>(listener, EqlSearchResponse::new, EsExecutors.DIRECT_EXECUTOR_SERVICE)
-                ),
-                log
+            planExecutor.eql(
+                cfg,
+                request.query(),
+                params,
+                wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), listener::onFailure)
             );
         }
     }
diff --git a/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt b/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt
index b04d28654f1d5..00c08096fd084 100644
--- a/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt
+++ b/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt
@@ -769,7 +769,7 @@ process where command_line regex "^.*?net.exe"
 regexSingleArgInsensitive
 process where command_line regex~ "^.*?net.exe"
 ;
-"regexp":{"command_line":{"value":"^.*?net.exe","flags_value":255,"case_insensitive":true
+"regexp":{"command_line":{"value":"^.*?net.exe","flags_value":65791,"case_insensitive":true
 ;
 
 regexMultiArg
@@ -781,7 +781,7 @@ process where command_line regex ("^.*?net.exe", "net\\.exe")
 regexMultiArgInsensitive
 process where command_line regex~ ("^.*?net.exe", "net\\.exe")
 ;
-"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe","flags_value":255,"case_insensitive":true
+"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe","flags_value":65791,"case_insensitive":true
 ;
 
 regexMultiMultiArgVariant
@@ -793,7 +793,7 @@ process where command_line regex ("^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\
 regexMultiMultiArgVariantInsensitive
 process where command_line regex~ ("^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+")
 ;
-"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+","flags_value":255,"case_insensitive":true
+"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+","flags_value":65791,"case_insensitive":true
 ;
 
 regexMultiArgWithScript
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
index e33f9b1c20527..1f7d03ba9d905 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -42,11 +43,11 @@ public Alias(Source source, String name, Expression child) {
         this(source, name, child, null);
     }
 
-    public Alias(Source source, String name, Expression child, NameId id) {
+    public Alias(Source source, String name, Expression child, @Nullable NameId id) {
         this(source, name, child, id, false);
     }
 
-    public Alias(Source source, String name, Expression child, NameId id, boolean synthetic) {
+    public Alias(Source source, String name, Expression child, @Nullable NameId id, boolean synthetic) {
         super(source, name, singletonList(child), id, synthetic);
         this.child = child;
     }
@@ -55,7 +56,7 @@ public Alias(Source source, String name, Expression child, NameId id, boolean sy
     /**
      * Old constructor from when this had a qualifier string. Still needed to not break serialization.
      */
-    private Alias(Source source, String name, String qualifier, Expression child, NameId id, boolean synthetic) {
+    private Alias(Source source, String name, String qualifier, Expression child, @Nullable NameId id, boolean synthetic) {
         this(source, name, child, id, synthetic);
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
index 05c414298fd33..45f42a754910d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.esql.core.expression;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
@@ -41,15 +42,15 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
     // can the attr be null - typically used in JOINs
     private final Nullability nullability;
 
-    public Attribute(Source source, String name, NameId id) {
+    public Attribute(Source source, String name, @Nullable NameId id) {
         this(source, name, Nullability.TRUE, id);
     }
 
-    public Attribute(Source source, String name, Nullability nullability, NameId id) {
+    public Attribute(Source source, String name, Nullability nullability, @Nullable NameId id) {
         this(source, name, nullability, id, false);
     }
 
-    public Attribute(Source source, String name, Nullability nullability, NameId id, boolean synthetic) {
+    public Attribute(Source source, String name, Nullability nullability, @Nullable NameId id, boolean synthetic) {
         super(source, name, emptyList(), id, synthetic);
         this.nullability = nullability;
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
index 767d2f45f90e4..4076acdb7e7b8 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
@@ -6,21 +6,25 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
 import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
-import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
 import java.io.IOException;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * Attribute for an ES field.
  * To differentiate between the different type of fields this class offers:
@@ -37,32 +41,31 @@ public class FieldAttribute extends TypedAttribute {
         FieldAttribute::readFrom
     );
 
-    private final FieldAttribute parent;
-    private final String path;
+    private final String parentName;
     private final EsField field;
 
     public FieldAttribute(Source source, String name, EsField field) {
         this(source, null, name, field);
     }
 
-    public FieldAttribute(Source source, FieldAttribute parent, String name, EsField field) {
-        this(source, parent, name, field, Nullability.TRUE, null, false);
+    public FieldAttribute(Source source, @Nullable String parentName, String name, EsField field) {
+        this(source, parentName, name, field, Nullability.TRUE, null, false);
     }
 
-    public FieldAttribute(Source source, FieldAttribute parent, String name, EsField field, boolean synthetic) {
-        this(source, parent, name, field, Nullability.TRUE, null, synthetic);
+    public FieldAttribute(Source source, @Nullable String parentName, String name, EsField field, boolean synthetic) {
+        this(source, parentName, name, field, Nullability.TRUE, null, synthetic);
     }
 
     public FieldAttribute(
         Source source,
-        FieldAttribute parent,
+        @Nullable String parentName,
         String name,
         EsField field,
         Nullability nullability,
-        NameId id,
+        @Nullable NameId id,
         boolean synthetic
     ) {
-        this(source, parent, name, field.getDataType(), field, nullability, id, synthetic);
+        this(source, parentName, name, field.getDataType(), field, nullability, id, synthetic);
     }
 
     /**
@@ -71,17 +74,16 @@ public FieldAttribute(
      */
     FieldAttribute(
         Source source,
-        FieldAttribute parent,
+        @Nullable String parentName,
         String name,
         DataType type,
         EsField field,
         Nullability nullability,
-        NameId id,
+        @Nullable NameId id,
         boolean synthetic
     ) {
         super(source, name, type, nullability, id, synthetic);
-        this.path = parent != null ? parent.name() : StringUtils.EMPTY;
-        this.parent = parent;
+        this.parentName = parentName;
         this.field = field;
     }
 
@@ -91,16 +93,16 @@ public FieldAttribute(
      */
     private FieldAttribute(
         Source source,
-        FieldAttribute parent,
+        @Nullable String parentName,
         String name,
         DataType type,
         EsField field,
-        String qualifier,
+        @Nullable String qualifier,
         Nullability nullability,
-        NameId id,
+        @Nullable NameId id,
         boolean synthetic
     ) {
-        this(source, parent, name, type, field, nullability, id, synthetic);
+        this(source, parentName, name, type, field, nullability, id, synthetic);
     }
 
     private FieldAttribute(StreamInput in) throws IOException {
@@ -114,8 +116,8 @@ private FieldAttribute(StreamInput in) throws IOException {
          */
         this(
             Source.readFrom((StreamInput & PlanStreamInput) in),
-            in.readOptionalWriteable(FieldAttribute::readFrom),
-            ((PlanStreamInput) in).readCachedString(),
+            readParentName(in),
+            readCachedStringWithVersionCheck(in),
             DataType.readFrom(in),
             EsField.readFrom(in),
             in.readOptionalString(),
@@ -129,8 +131,8 @@ private FieldAttribute(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) {
             Source.EMPTY.writeTo(out);
-            out.writeOptionalWriteable(parent);
-            ((PlanStreamOutput) out).writeCachedString(name());
+            writeParentName(out);
+            writeCachedStringWithVersionCheck(out, name());
             dataType().writeTo(out);
             field.writeTo(out);
             // We used to write the qualifier here. We can still do if needed in the future.
@@ -145,6 +147,26 @@ public static FieldAttribute readFrom(StreamInput in) throws IOException {
         return ((PlanStreamInput) in).readAttributeWithCache(FieldAttribute::new);
     }
 
+    private void writeParentName(StreamOutput out) throws IOException {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED)) {
+            ((PlanStreamOutput) out).writeOptionalCachedString(parentName);
+        } else {
+            // Previous versions only used the parent field attribute to retrieve the parent's name, so we can use just any
+            // fake FieldAttribute here as long as the name is correct.
+            FieldAttribute fakeParent = parentName() == null ? null : new FieldAttribute(Source.EMPTY, parentName(), field());
+            out.writeOptionalWriteable(fakeParent);
+        }
+    }
+
+    private static String readParentName(StreamInput in) throws IOException {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED)) {
+            return ((PlanStreamInput) in).readOptionalCachedString();
+        }
+
+        FieldAttribute parent = in.readOptionalWriteable(FieldAttribute::readFrom);
+        return parent == null ? null : parent.name();
+    }
+
     @Override
     public String getWriteableName() {
         return ENTRY.name;
@@ -152,15 +174,22 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<FieldAttribute> info() {
-        return NodeInfo.create(this, FieldAttribute::new, parent, name(), dataType(), field, (String) null, nullable(), id(), synthetic());
-    }
-
-    public FieldAttribute parent() {
-        return parent;
+        return NodeInfo.create(
+            this,
+            FieldAttribute::new,
+            parentName,
+            name(),
+            dataType(),
+            field,
+            (String) null,
+            nullable(),
+            id(),
+            synthetic()
+        );
     }
 
-    public String path() {
-        return path;
+    public String parentName() {
+        return parentName;
     }
 
     /**
@@ -174,7 +203,7 @@ public String fieldName() {
         if ((synthetic() || name().startsWith(SYNTHETIC_ATTRIBUTE_NAME_PREFIX)) == false) {
             return name();
         }
-        return Strings.hasText(path) ? path + "." + field.getName() : field.getName();
+        return Strings.hasText(parentName) ? parentName + "." + field.getName() : field.getName();
     }
 
     public EsField.Exact getExactInfo() {
@@ -190,13 +219,13 @@ public FieldAttribute exactAttribute() {
     }
 
     private FieldAttribute innerField(EsField type) {
-        return new FieldAttribute(source(), this, name() + "." + type.getName(), type, nullable(), id(), synthetic());
+        return new FieldAttribute(source(), name(), name() + "." + type.getName(), type, nullable(), id(), synthetic());
     }
 
     @Override
     protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
         // Ignore `type`, this must be the same as the field's type.
-        return new FieldAttribute(source, parent, name, field, nullability, id, synthetic);
+        return new FieldAttribute(source, parentName, name, field, nullability, id, synthetic);
     }
 
     @Override
@@ -206,13 +235,13 @@ public Attribute withDataType(DataType type) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), path, field);
+        return Objects.hash(super.hashCode(), parentName, field);
     }
 
     @Override
     public boolean equals(Object obj) {
         return super.equals(obj)
-            && Objects.equals(path, ((FieldAttribute) obj).path)
+            && Objects.equals(parentName, ((FieldAttribute) obj).parentName)
             && Objects.equals(field, ((FieldAttribute) obj).field);
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
index 539c55ba341cf..3641812cd6cad 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
@@ -59,7 +60,7 @@ public MetadataAttribute(
         String name,
         DataType dataType,
         Nullability nullability,
-        NameId id,
+        @Nullable NameId id,
         boolean synthetic,
         boolean searchable
     ) {
@@ -79,9 +80,9 @@ private MetadataAttribute(
         Source source,
         String name,
         DataType dataType,
-        String qualifier,
+        @Nullable String qualifier,
         Nullability nullability,
-        NameId id,
+        @Nullable NameId id,
         boolean synthetic,
         boolean searchable
     ) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
index ba467910bed0d..3b018f09e5ebd 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 import java.util.ArrayList;
@@ -32,11 +33,11 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
     private final NameId id;
     private final boolean synthetic;
 
-    public NamedExpression(Source source, String name, List<Expression> children, NameId id) {
+    public NamedExpression(Source source, String name, List<Expression> children, @Nullable NameId id) {
         this(source, name, children, id, false);
     }
 
-    public NamedExpression(Source source, String name, List<Expression> children, NameId id, boolean synthetic) {
+    public NamedExpression(Source source, String name, List<Expression> children, @Nullable NameId id, boolean synthetic) {
         super(source, children);
         this.name = name;
         this.id = id == null ? new NameId() : id;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
index 504e1eae8d880..3626c5d26f235 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -31,7 +32,14 @@ public ReferenceAttribute(Source source, String name, DataType dataType) {
         this(source, name, dataType, Nullability.FALSE, null, false);
     }
 
-    public ReferenceAttribute(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) {
+    public ReferenceAttribute(
+        Source source,
+        String name,
+        DataType dataType,
+        Nullability nullability,
+        @Nullable NameId id,
+        boolean synthetic
+    ) {
         super(source, name, dataType, nullability, id, synthetic);
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
index ab05a71b0e1c6..b817ec17c7bda 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
@@ -155,18 +155,19 @@ public static TypeResolution isNotNullAndFoldable(Expression e, String operation
         return resolution;
     }
 
-    public static TypeResolution isNotFoldable(Expression e, String operationName, ParamOrdinal paramOrd) {
-        if (e.foldable()) {
+    public static TypeResolution isNotNull(Expression e, String operationName, ParamOrdinal paramOrd) {
+        if (e.dataType() == DataType.NULL) {
             return new TypeResolution(
                 format(
                     null,
-                    "{}argument of [{}] must be a table column, found constant [{}]",
+                    "{}argument of [{}] cannot be null, received [{}]",
                     paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ",
                     operationName,
                     Expressions.name(e)
                 )
             );
         }
+
         return TypeResolution.TYPE_RESOLVED;
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java
index 0350abef99992..f8a041110798c 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
@@ -15,7 +16,14 @@ public abstract class TypedAttribute extends Attribute {
 
     private final DataType dataType;
 
-    protected TypedAttribute(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) {
+    protected TypedAttribute(
+        Source source,
+        String name,
+        DataType dataType,
+        Nullability nullability,
+        @Nullable NameId id,
+        boolean synthetic
+    ) {
         super(source, name, nullability, id, synthetic);
         this.dataType = dataType;
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
index d8a35adcbffde..a971a15a23c86 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.esql.core.expression;
 
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
 import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -33,7 +34,7 @@ public UnresolvedAttribute(Source source, String name, String unresolvedMessage)
     }
 
     @SuppressWarnings("this-escape")
-    public UnresolvedAttribute(Source source, String name, NameId id, String unresolvedMessage, Object resolutionMetadata) {
+    public UnresolvedAttribute(Source source, String name, @Nullable NameId id, String unresolvedMessage, Object resolutionMetadata) {
         super(source, name, id);
         this.customMessage = unresolvedMessage != null;
         this.unresolvedMsg = unresolvedMessage == null ? errorMessage(name(), null) : unresolvedMessage;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/Like.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/Like.java
deleted file mode 100644
index 6d8ce8cbdf47f..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/Like.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
-
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-import java.io.IOException;
-
-public class Like extends RegexMatch<LikePattern> {
-
-    public Like(Source source, Expression left, LikePattern pattern) {
-        this(source, left, pattern, false);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public String getWriteableName() {
-        throw new UnsupportedOperationException();
-    }
-
-    public Like(Source source, Expression left, LikePattern pattern, boolean caseInsensitive) {
-        super(source, left, pattern, caseInsensitive);
-    }
-
-    @Override
-    protected NodeInfo<Like> info() {
-        return NodeInfo.create(this, Like::new, field(), pattern(), caseInsensitive());
-    }
-
-    @Override
-    protected Like replaceChild(Expression newLeft) {
-        return new Like(source(), newLeft, pattern(), caseInsensitive());
-    }
-
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/LikePattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/LikePattern.java
deleted file mode 100644
index 52ce2636e914b..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/LikePattern.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
-
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
-import org.apache.lucene.util.automaton.Operations;
-import org.elasticsearch.xpack.esql.core.util.StringUtils;
-
-import java.util.Objects;
-
-/**
- * A SQL 'like' pattern.
- * Similar to basic regex, supporting '_' instead of '?' and '%' instead of '*'.
- * <p>
- * Allows escaping based on a regular char.
- *
- * To prevent conflicts with ES, the string and char must be validated to not contain '*'.
- */
-public class LikePattern extends AbstractStringPattern {
-
-    private final String pattern;
-    private final char escape;
-
-    private final String regex;
-    private final String wildcard;
-    private final String indexNameWildcard;
-
-    public LikePattern(String pattern, char escape) {
-        this.pattern = pattern;
-        this.escape = escape;
-        // early initialization to force string validation
-        this.regex = StringUtils.likeToJavaPattern(pattern, escape);
-        this.wildcard = StringUtils.likeToLuceneWildcard(pattern, escape);
-        this.indexNameWildcard = StringUtils.likeToIndexWildcard(pattern, escape);
-    }
-
-    public String pattern() {
-        return pattern;
-    }
-
-    public char escape() {
-        return escape;
-    }
-
-    @Override
-    public Automaton createAutomaton() {
-        Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard));
-        return MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
-    }
-
-    @Override
-    public String asJavaRegex() {
-        return regex;
-    }
-
-    /**
-     * Returns the pattern in (Lucene) wildcard format.
-     */
-    public String asLuceneWildcard() {
-        return wildcard;
-    }
-
-    /**
-     * Returns the pattern in (IndexNameExpressionResolver) wildcard format.
-     */
-    public String asIndexNameWildcard() {
-        return indexNameWildcard;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(pattern, escape);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-
-        LikePattern other = (LikePattern) obj;
-        return Objects.equals(pattern, other.pattern) && escape == other.escape;
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java
index f437dc5819dcb..4e559f564acb1 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
 
 import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 
 import java.util.Objects;
@@ -21,7 +22,10 @@ public RLikePattern(String regexpPattern) {
 
     @Override
     public Automaton createAutomaton() {
-        return new RegExp(regexpPattern).toAutomaton();
+        return Operations.determinize(
+            new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton(),
+            Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java
index 7cedbc4742138..3e9cbf92727c2 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java
@@ -9,7 +9,6 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
@@ -39,8 +38,7 @@ public String pattern() {
 
     @Override
     public Automaton createAutomaton() {
-        Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard));
-        return MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        return WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
index 176250222512b..366630eadb5fe 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
@@ -66,9 +65,6 @@ public static Query doTranslate(RegexMatch e, TranslatorHandler handler) {
         }
 
         private static Query translateField(RegexMatch e, String targetFieldName) {
-            if (e instanceof Like l) {
-                return new WildcardQuery(e.source(), targetFieldName, l.pattern().asLuceneWildcard(), l.caseInsensitive());
-            }
             if (e instanceof WildcardLike l) {
                 return new WildcardQuery(e.source(), targetFieldName, l.pattern().asLuceneWildcard(), l.caseInsensitive());
             }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
index 639d8ed68d0a3..d84a471815a9a 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
@@ -14,4 +14,5 @@
 public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin {
     public static final FeatureFlag DATE_NANOS_FEATURE_FLAG = new FeatureFlag("esql_date_nanos");
 
+    public static final FeatureFlag SEMANTIC_TEXT_FEATURE_FLAG = new FeatureFlag("esql_semantic_text");
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
index c0092caeb9d5d..5041c96128a1e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
@@ -14,8 +14,6 @@
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.math.BigInteger;
@@ -32,6 +30,8 @@
 
 import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableMap;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
 
 public enum DataType {
     /**
@@ -194,7 +194,14 @@ public enum DataType {
      * inside alongside time-series aggregations. These fields are not parsable from the
      * mapping and should be hidden from users.
      */
-    PARTIAL_AGG(builder().esType("partial_agg").unknownSize());
+    PARTIAL_AGG(builder().esType("partial_agg").unknownSize()),
+    /**
+     * String fields that are split into chunks, where each chunk has attached embeddings
+     * used for semantic search. Generally ESQL only sees {@code semantic_text} fields when
+     * loaded from the index and ESQL will load these fields as strings without their attached
+     * chunks or embeddings.
+     */
+    SEMANTIC_TEXT(builder().esType("semantic_text").unknownSize());
 
     /**
      * Types that are actively being built. These types are not returned
@@ -203,7 +210,8 @@ public enum DataType {
      * check that sending them to a function produces a sane error message.
      */
     public static final Map<DataType, FeatureFlag> UNDER_CONSTRUCTION = Map.ofEntries(
-        Map.entry(DATE_NANOS, EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG)
+        Map.entry(DATE_NANOS, EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+        Map.entry(SEMANTIC_TEXT, EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG)
     );
 
     private final String typeName;
@@ -266,6 +274,8 @@ public enum DataType {
         .sorted(Comparator.comparing(DataType::typeName))
         .toList();
 
+    private static final Collection<DataType> STRING_TYPES = DataType.types().stream().filter(DataType::isString).toList();
+
     private static final Map<String, DataType> NAME_TO_TYPE = TYPES.stream().collect(toUnmodifiableMap(DataType::typeName, t -> t));
 
     private static final Map<String, DataType> ES_TO_TYPE;
@@ -292,6 +302,10 @@ public static Collection<DataType> types() {
         return TYPES;
     }
 
+    public static Collection<DataType> stringTypes() {
+        return STRING_TYPES;
+    }
+
     /**
      * Resolve a type from a name. This name is sometimes user supplied,
      * like in the case of {@code ::<typename>} and is sometimes the name
@@ -425,6 +439,10 @@ public static boolean isRepresentable(DataType t) {
             && t.isCounter() == false;
     }
 
+    public static boolean isCounter(DataType t) {
+        return t == COUNTER_DOUBLE || t == COUNTER_INTEGER || t == COUNTER_LONG;
+    }
+
     public static boolean isSpatialPoint(DataType t) {
         return t == GEO_POINT || t == CARTESIAN_POINT;
     }
@@ -437,6 +455,10 @@ public static boolean isSpatial(DataType t) {
         return t == GEO_POINT || t == CARTESIAN_POINT || t == GEO_SHAPE || t == CARTESIAN_SHAPE;
     }
 
+    public static boolean isSortable(DataType t) {
+        return false == (t == SOURCE || isCounter(t) || isSpatial(t));
+    }
+
     public String nameUpper() {
         return name;
     }
@@ -521,12 +543,12 @@ public DataType counter() {
     }
 
     public void writeTo(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(typeName);
+        writeCachedStringWithVersionCheck(out, typeName);
     }
 
     public static DataType readFrom(StreamInput in) throws IOException {
         // TODO: Use our normal enum serialization pattern
-        return readFrom(((PlanStreamInput) in).readCachedString());
+        return readFrom(readCachedStringWithVersionCheck(in));
     }
 
     /**
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
index 78b395503e700..7c91a506697c1 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
@@ -30,11 +30,9 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
@@ -62,7 +60,7 @@ public static Converter converterFor(DataType from, DataType to) {
             return DefaultConverter.TO_NULL;
         }
         // proper converters
-        if (to == KEYWORD || to == TEXT) {
+        if (isString(to)) {
             return conversionToString(from);
         }
         if (to == LONG) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
index 7c4b98c5af84e..3a81ec2a6f17d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
@@ -8,12 +8,13 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * Information about a field in an ES index with the {@code date} type
  */
@@ -28,12 +29,12 @@ private DateEsField(String name, DataType dataType, Map<String, EsField> propert
     }
 
     protected DateEsField(StreamInput in) throws IOException {
-        this(((PlanStreamInput) in).readCachedString(), DataType.DATETIME, in.readImmutableMap(EsField::readFrom), in.readBoolean());
+        this(readCachedStringWithVersionCheck(in), DataType.DATETIME, in.readImmutableMap(EsField::readFrom), in.readBoolean());
     }
 
     @Override
     public void writeContent(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(getName());
+        writeCachedStringWithVersionCheck(out, getName());
         out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
         out.writeBoolean(isAggregatable());
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
index 6235176d82de6..47dadcbb11de2 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
@@ -18,6 +18,9 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * Information about a field in an ES index.
  */
@@ -60,7 +63,7 @@ public EsField(String name, DataType esDataType, Map<String, EsField> properties
     }
 
     public EsField(StreamInput in) throws IOException {
-        this.name = ((PlanStreamInput) in).readCachedString();
+        this.name = readCachedStringWithVersionCheck(in);
         this.esDataType = readDataType(in);
         this.properties = in.readImmutableMap(EsField::readFrom);
         this.aggregatable = in.readBoolean();
@@ -68,7 +71,7 @@ public EsField(StreamInput in) throws IOException {
     }
 
     private DataType readDataType(StreamInput in) throws IOException {
-        String name = ((PlanStreamInput) in).readCachedString();
+        String name = readCachedStringWithVersionCheck(in);
         if (in.getTransportVersion().before(TransportVersions.ESQL_NESTED_UNSUPPORTED) && name.equalsIgnoreCase("NESTED")) {
             /*
              * The "nested" data type existed in older versions of ESQL but was
@@ -98,7 +101,7 @@ public void writeTo(StreamOutput out) throws IOException {
      * This needs to be overridden by subclasses for specific serialization
      */
     public void writeContent(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(name);
+        writeCachedStringWithVersionCheck(out, name);
         esDataType.writeTo(out);
         out.writeMap(properties, (o, x) -> x.writeTo(out));
         out.writeBoolean(aggregatable);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
index 40825af56ccfe..f83e4652ebebd 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.Map;
@@ -20,6 +18,9 @@
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * Representation of field mapped differently across indices.
  * Used during mapping discovery only.
@@ -54,7 +55,7 @@ private InvalidMappedField(String name, String errorMessage, Map<String, EsField
     }
 
     protected InvalidMappedField(StreamInput in) throws IOException {
-        this(((PlanStreamInput) in).readCachedString(), in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom));
+        this(readCachedStringWithVersionCheck(in), in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom));
     }
 
     public Set<DataType> types() {
@@ -63,7 +64,7 @@ public Set<DataType> types() {
 
     @Override
     public void writeContent(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(getName());
+        writeCachedStringWithVersionCheck(out, getName());
         out.writeString(errorMessage);
         out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
index 48995bafec451..8b88884a0ce17 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
@@ -8,8 +8,6 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -17,6 +15,8 @@
 import java.util.Objects;
 
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
 
 /**
  * Information about a field in an ES index with the {@code keyword} type.
@@ -61,7 +61,7 @@ protected KeywordEsField(
 
     public KeywordEsField(StreamInput in) throws IOException {
         this(
-            ((PlanStreamInput) in).readCachedString(),
+            readCachedStringWithVersionCheck(in),
             KEYWORD,
             in.readImmutableMap(EsField::readFrom),
             in.readBoolean(),
@@ -73,7 +73,7 @@ public KeywordEsField(StreamInput in) throws IOException {
 
     @Override
     public void writeContent(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(getName());
+        writeCachedStringWithVersionCheck(out, getName());
         out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
         out.writeBoolean(isAggregatable());
         out.writeInt(precision);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java
index 522cb682c0943..0d7f9ee425d6a 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -19,6 +17,9 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * During IndexResolution it could occur that the same field is mapped to different types in different indices.
  * The class MultiTypeEfField.UnresolvedField holds that information and allows for later resolution of the field
@@ -39,7 +40,7 @@ public MultiTypeEsField(String name, DataType dataType, boolean aggregatable, Ma
 
     protected MultiTypeEsField(StreamInput in) throws IOException {
         this(
-            ((PlanStreamInput) in).readCachedString(),
+            readCachedStringWithVersionCheck(in),
             DataType.readFrom(in),
             in.readBoolean(),
             in.readImmutableMap(i -> i.readNamedWriteable(Expression.class))
@@ -48,7 +49,7 @@ protected MultiTypeEsField(StreamInput in) throws IOException {
 
     @Override
     public void writeContent(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(getName());
+        writeCachedStringWithVersionCheck(out, getName());
         getDataType().writeTo(out);
         out.writeBoolean(isAggregatable());
         out.writeMap(getIndexToConversionExpressions(), (o, v) -> out.writeNamedWriteable(v));
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
index c6c494ef289bb..ed0d32a7696eb 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.Map;
@@ -19,6 +17,8 @@
 
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
 
 /**
  * Information about a field in an es index with the {@code text} type.
@@ -34,12 +34,12 @@ public TextEsField(String name, Map<String, EsField> properties, boolean hasDocV
     }
 
     protected TextEsField(StreamInput in) throws IOException {
-        this(((PlanStreamInput) in).readCachedString(), in.readImmutableMap(EsField::readFrom), in.readBoolean(), in.readBoolean());
+        this(readCachedStringWithVersionCheck(in), in.readImmutableMap(EsField::readFrom), in.readBoolean(), in.readBoolean());
     }
 
     @Override
     public void writeContent(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(getName());
+        writeCachedStringWithVersionCheck(out, getName());
         out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
         out.writeBoolean(isAggregatable());
         out.writeBoolean(isAlias());
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
index 980620cb98847..02ce741243c20 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
@@ -8,14 +8,15 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * Information about a field in an ES index that cannot be supported by ESQL.
  * All the subfields (properties) of an unsupported type are also be unsupported.
@@ -37,8 +38,8 @@ public UnsupportedEsField(String name, String originalType, String inherited, Ma
 
     public UnsupportedEsField(StreamInput in) throws IOException {
         this(
-            ((PlanStreamInput) in).readCachedString(),
-            ((PlanStreamInput) in).readCachedString(),
+            readCachedStringWithVersionCheck(in),
+            readCachedStringWithVersionCheck(in),
             in.readOptionalString(),
             in.readImmutableMap(EsField::readFrom)
         );
@@ -46,8 +47,8 @@ public UnsupportedEsField(StreamInput in) throws IOException {
 
     @Override
     public void writeContent(StreamOutput out) throws IOException {
-        ((PlanStreamOutput) out).writeCachedString(getName());
-        ((PlanStreamOutput) out).writeCachedString(getOriginalType());
+        writeCachedStringWithVersionCheck(out, getName());
+        writeCachedStringWithVersionCheck(out, getOriginalType());
         out.writeOptionalString(getInherited());
         out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java
index 48b5fd1605edf..8bfcf4ca5c405 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java
@@ -79,4 +79,19 @@ public static int mapSize(int size) {
         }
         return (int) (size / 0.75f + 1f);
     }
+
+    @SafeVarargs
+    @SuppressWarnings("varargs")
+    public static <T> List<T> nullSafeList(T... entries) {
+        if (entries == null || entries.length == 0) {
+            return emptyList();
+        }
+        List<T> list = new ArrayList<>(entries.length);
+        for (T entry : entries) {
+            if (entry != null) {
+                list.add(entry);
+            }
+        }
+        return list;
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
index 826b0cbfa3498..e8ccae3429001 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.core.util;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -49,4 +50,13 @@ public interface PlanStreamInput {
     <A extends EsField> A readEsFieldWithCache() throws IOException;
 
     String readCachedString() throws IOException;
+
+    static String readCachedStringWithVersionCheck(StreamInput planStreamInput) throws IOException {
+        if (planStreamInput.getTransportVersion().before(TransportVersions.ESQL_CACHED_STRING_SERIALIZATION)) {
+            return planStreamInput.readString();
+        }
+        return ((PlanStreamInput) planStreamInput).readCachedString();
+    }
+
+    String readOptionalCachedString() throws IOException;
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
index e4797411c3796..fb4af33d2fd60 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.esql.core.util;
 
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 
@@ -33,4 +35,14 @@ public interface PlanStreamOutput {
     boolean writeEsFieldCacheHeader(EsField field) throws IOException;
 
     void writeCachedString(String field) throws IOException;
+
+    static void writeCachedStringWithVersionCheck(StreamOutput planStreamOutput, String string) throws IOException {
+        if (planStreamOutput.getTransportVersion().before(TransportVersions.ESQL_CACHED_STRING_SERIALIZATION)) {
+            planStreamOutput.writeString(string);
+        } else {
+            ((PlanStreamOutput) planStreamOutput).writeCachedString(string);
+        }
+    }
+
+    void writeOptionalCachedString(String str) throws IOException;
 }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FieldAttributeTestUtils.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FieldAttributeTestUtils.java
index 1662b7f973c9d..c7e5056ed0267 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FieldAttributeTestUtils.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FieldAttributeTestUtils.java
@@ -12,9 +12,9 @@
 import org.elasticsearch.xpack.esql.core.type.EsField;
 
 public class FieldAttributeTestUtils {
-    public static final FieldAttribute newFieldAttributeWithType(
+    public static FieldAttribute newFieldAttributeWithType(
         Source source,
-        FieldAttribute parent,
+        String parentName,
         String name,
         DataType type,
         EsField field,
@@ -22,6 +22,6 @@ public static final FieldAttribute newFieldAttributeWithType(
         NameId id,
         boolean synthetic
     ) {
-        return new FieldAttribute(source, parent, name, type, field, nullability, id, synthetic);
+        return new FieldAttribute(source, parentName, name, type, field, nullability, id, synthetic);
     }
 }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java
index 43cae475cff7e..c361b7e3726ed 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java
@@ -12,78 +12,78 @@
 
 public class StringPatternTests extends ESTestCase {
 
-    private LikePattern like(String pattern, char escape) {
-        return new LikePattern(pattern, escape);
+    private WildcardPattern like(String pattern) {
+        return new WildcardPattern(pattern);
     }
 
     private RLikePattern rlike(String pattern) {
         return new RLikePattern(pattern);
     }
 
-    private boolean matchesAll(String pattern, char escape) {
-        return like(pattern, escape).matchesAll();
+    private boolean likeMatchesAll(String pattern) {
+        return like(pattern).matchesAll();
     }
 
-    private boolean exactMatch(String pattern, char escape) {
-        String escaped = pattern.replace(Character.toString(escape), StringUtils.EMPTY);
-        return escaped.equals(like(pattern, escape).exactMatch());
+    private boolean likeExactMatch(String pattern) {
+        String escaped = pattern.replace("\\", StringUtils.EMPTY);
+        return escaped.equals(like(pattern).exactMatch());
     }
 
-    private boolean matchesAll(String pattern) {
+    private boolean rlikeMatchesAll(String pattern) {
         return rlike(pattern).matchesAll();
     }
 
-    private boolean exactMatch(String pattern) {
+    private boolean rlikeExactMatch(String pattern) {
         return pattern.equals(rlike(pattern).exactMatch());
     }
 
-    public void testWildcardMatchAll() throws Exception {
-        assertTrue(matchesAll("%", '0'));
-        assertTrue(matchesAll("%%", '0'));
+    public void testWildcardMatchAll() {
+        assertTrue(likeMatchesAll("*"));
+        assertTrue(likeMatchesAll("**"));
 
-        assertFalse(matchesAll("a%", '0'));
-        assertFalse(matchesAll("%_", '0'));
-        assertFalse(matchesAll("%_%_%", '0'));
-        assertFalse(matchesAll("_%", '0'));
-        assertFalse(matchesAll("0%", '0'));
+        assertFalse(likeMatchesAll("a*"));
+        assertFalse(likeMatchesAll("*?"));
+        assertFalse(likeMatchesAll("*?*?*"));
+        assertFalse(likeMatchesAll("?*"));
+        assertFalse(likeMatchesAll("\\*"));
     }
 
-    public void testRegexMatchAll() throws Exception {
-        assertTrue(matchesAll(".*"));
-        assertTrue(matchesAll(".*.*"));
-        assertTrue(matchesAll(".*.?"));
-        assertTrue(matchesAll(".?.*"));
-        assertTrue(matchesAll(".*.?.*"));
+    public void testRegexMatchAll() {
+        assertTrue(rlikeMatchesAll(".*"));
+        assertTrue(rlikeMatchesAll(".*.*"));
+        assertTrue(rlikeMatchesAll(".*.?"));
+        assertTrue(rlikeMatchesAll(".?.*"));
+        assertTrue(rlikeMatchesAll(".*.?.*"));
 
-        assertFalse(matchesAll("..*"));
-        assertFalse(matchesAll("ab."));
-        assertFalse(matchesAll("..?"));
+        assertFalse(rlikeMatchesAll("..*"));
+        assertFalse(rlikeMatchesAll("ab."));
+        assertFalse(rlikeMatchesAll("..?"));
     }
 
-    public void testWildcardExactMatch() throws Exception {
-        assertTrue(exactMatch("0%", '0'));
-        assertTrue(exactMatch("0_", '0'));
-        assertTrue(exactMatch("123", '0'));
-        assertTrue(exactMatch("1230_", '0'));
-        assertTrue(exactMatch("1230_321", '0'));
-
-        assertFalse(exactMatch("%", '0'));
-        assertFalse(exactMatch("%%", '0'));
-        assertFalse(exactMatch("a%", '0'));
-        assertFalse(exactMatch("a_", '0'));
+    public void testWildcardExactMatch() {
+        assertTrue(likeExactMatch("\\*"));
+        assertTrue(likeExactMatch("\\?"));
+        assertTrue(likeExactMatch("123"));
+        assertTrue(likeExactMatch("123\\?"));
+        assertTrue(likeExactMatch("123\\?321"));
+
+        assertFalse(likeExactMatch("*"));
+        assertFalse(likeExactMatch("**"));
+        assertFalse(likeExactMatch("a*"));
+        assertFalse(likeExactMatch("a?"));
     }
 
-    public void testRegexExactMatch() throws Exception {
-        assertFalse(exactMatch(".*"));
-        assertFalse(exactMatch(".*.*"));
-        assertFalse(exactMatch(".*.?"));
-        assertFalse(exactMatch(".?.*"));
-        assertFalse(exactMatch(".*.?.*"));
-        assertFalse(exactMatch("..*"));
-        assertFalse(exactMatch("ab."));
-        assertFalse(exactMatch("..?"));
-
-        assertTrue(exactMatch("abc"));
-        assertTrue(exactMatch("12345"));
+    public void testRegexExactMatch() {
+        assertFalse(rlikeExactMatch(".*"));
+        assertFalse(rlikeExactMatch(".*.*"));
+        assertFalse(rlikeExactMatch(".*.?"));
+        assertFalse(rlikeExactMatch(".?.*"));
+        assertFalse(rlikeExactMatch(".*.?.*"));
+        assertFalse(rlikeExactMatch("..*"));
+        assertFalse(rlikeExactMatch("ab."));
+        assertFalse(rlikeExactMatch("..?"));
+
+        assertTrue(rlikeExactMatch("abc"));
+        assertTrue(rlikeExactMatch("12345"));
     }
 }
diff --git a/x-pack/plugin/esql/arrow/build.gradle b/x-pack/plugin/esql/arrow/build.gradle
index 20c877a12bf0d..fac0bd0a77452 100644
--- a/x-pack/plugin/esql/arrow/build.gradle
+++ b/x-pack/plugin/esql/arrow/build.gradle
@@ -26,6 +26,7 @@ dependencies {
 
   testImplementation project(':test:framework')
   testImplementation('org.apache.arrow:arrow-memory-unsafe:16.1.0')
+  testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}")
 }
 
 tasks.named("dependencyLicenses").configure {
diff --git a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java
index 7a8328060a390..208d3308d508b 100644
--- a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java
+++ b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java
@@ -17,6 +17,7 @@
 import org.apache.arrow.vector.ipc.message.MessageSerializer;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionListener;
@@ -44,6 +45,7 @@ public class ArrowResponse implements ChunkedRestResponseBodyPart, Releasable {
     public static class Column {
         private final BlockConverter converter;
         private final String name;
+        private boolean multivalued;
 
         public Column(String esqlType, String name) {
             this.converter = ESQL_CONVERTERS.get(esqlType);
@@ -61,20 +63,24 @@ public Column(String esqlType, String name) {
     public ArrowResponse(List<Column> columns, List<Page> pages) {
         this.columns = columns;
 
+        // Find multivalued columns
+        int colSize = columns.size();
+        for (int col = 0; col < colSize; col++) {
+            for (Page page : pages) {
+                if (page.getBlock(col).mayHaveMultivaluedFields()) {
+                    columns.get(col).multivalued = true;
+                    break;
+                }
+            }
+        }
+
         currentSegment = new SchemaResponse(this);
         List<ResponseSegment> rest = new ArrayList<>(pages.size());
-        for (int p = 0; p < pages.size(); p++) {
-            var page = pages.get(p);
+
+        for (Page page : pages) {
             rest.add(new PageResponse(this, page));
-            // Multivalued fields are not supported yet.
-            for (int b = 0; b < page.getBlockCount(); b++) {
-                if (page.getBlock(b).mayHaveMultivaluedFields()) {
-                    throw new IllegalArgumentException(
-                        "ES|QL response field [" + columns.get(b).name + "] is multi-valued. This isn't supported yet by the Arrow format"
-                    );
-                }
-            }
         }
+
         rest.add(new EndResponse(this));
         segments = rest.iterator();
     }
@@ -185,6 +191,9 @@ public void close() {}
      * @see <a href="https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format">IPC Streaming Format</a>
      */
     private static class SchemaResponse extends ResponseSegment {
+
+        private static final FieldType LIST_FIELD_TYPE = FieldType.nullable(MinorType.LIST.getType());
+
         private boolean done = false;
 
         SchemaResponse(ArrowResponse response) {
@@ -204,7 +213,20 @@ protected void encodeChunk(int sizeHint, RecyclerBytesStreamOutput out) throws I
         }
 
         private Schema arrowSchema() {
-            return new Schema(response.columns.stream().map(c -> new Field(c.name, c.converter.arrowFieldType(), List.of())).toList());
+            return new Schema(response.columns.stream().map(c -> {
+                var fieldType = c.converter.arrowFieldType();
+                if (c.multivalued) {
+                    // A variable-sized list is a vector of offsets and a child vector of values
+                    // See https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout
+                    var listType = new FieldType(true, LIST_FIELD_TYPE.getType(), null, fieldType.getMetadata());
+                    // Value vector is non-nullable (ES|QL multivalues cannot contain nulls).
+                    var valueType = new FieldType(false, fieldType.getType(), fieldType.getDictionary(), null);
+                    // The nested vector is named "$data$", following what the Arrow/Java library does.
+                    return new Field(c.name, listType, List.of(new Field("$data$", valueType, null)));
+                } else {
+                    return new Field(c.name, fieldType, null);
+                }
+            }).toList());
         }
     }
 
@@ -257,7 +279,14 @@ protected void encodeChunk(int sizeHint, RecyclerBytesStreamOutput out) throws I
 
                 @Override
                 public void write(ArrowBuf buffer) throws IOException {
-                    extraPosition += bufWriters.get(bufIdx++).write(out);
+                    var len = bufWriters.get(bufIdx++).write(out);
+                    // Consistency check
+                    if (len != buffer.writerIndex()) {
+                        throw new IllegalStateException(
+                            "Buffer [" + (bufIdx - 1) + "]: wrote [" + len + "] bytes, but expected [" + buffer.writerIndex() + "]"
+                        );
+                    }
+                    extraPosition += len;
                 }
 
                 @Override
@@ -277,11 +306,26 @@ public long align() throws IOException {
 
             // Create Arrow buffers for each of the blocks in this page
             for (int b = 0; b < page.getBlockCount(); b++) {
-                var converter = response.columns.get(b).converter;
+                var column = response.columns.get(b);
+                var converter = column.converter;
 
                 Block block = page.getBlock(b);
-                nodes.add(new ArrowFieldNode(block.getPositionCount(), converter.nullValuesCount(block)));
-                converter.convert(block, bufs, bufWriters);
+                if (column.multivalued) {
+                    // List node.
+                    nodes.add(new ArrowFieldNode(block.getPositionCount(), converter.nullValuesCount(block)));
+                    // Value vector, does not contain nulls.
+                    nodes.add(new ArrowFieldNode(BlockConverter.valueCount(block), 0));
+                } else {
+                    nodes.add(new ArrowFieldNode(block.getPositionCount(), converter.nullValuesCount(block)));
+                }
+                converter.convert(block, column.multivalued, bufs, bufWriters);
+            }
+
+            // Consistency check
+            if (bufs.size() != bufWriters.size()) {
+                throw new IllegalStateException(
+                    "Inconsistent Arrow buffers: [" + bufs.size() + "] buffers and [" + bufWriters.size() + "] writers"
+                );
             }
 
             // Create the batch and serialize it
diff --git a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/BlockConverter.java b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/BlockConverter.java
index 0a65792ab8e13..2a305cfdbc503 100644
--- a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/BlockConverter.java
+++ b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/BlockConverter.java
@@ -71,10 +71,11 @@ public interface BufWriter {
     /**
      * Convert a block into Arrow buffers.
      * @param block the ESQL block
+     * @param multivalued is this column multivalued? This block may not, but some blocks in that column are.
      * @param bufs arrow buffers, used to track sizes
      * @param bufWriters buffer writers, that will do the actual work of writing the data
      */
-    public abstract void convert(Block block, List<ArrowBuf> bufs, List<BufWriter> bufWriters);
+    public abstract void convert(Block block, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters);
 
     /**
      * Conversion of Double blocks
@@ -86,28 +87,31 @@ public AsFloat64(String esqlType) {
         }
 
         @Override
-        public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
+        public void convert(Block b, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             DoubleBlock block = (DoubleBlock) b;
 
-            accumulateVectorValidity(bufs, bufWriters, block);
+            if (multivalued) {
+                addListOffsets(bufs, bufWriters, block);
+            }
+            accumulateVectorValidity(bufs, bufWriters, block, multivalued);
 
-            bufs.add(dummyArrowBuf(vectorLength(block)));
+            bufs.add(dummyArrowBuf(vectorByteSize(block)));
             bufWriters.add(out -> {
                 if (block.areAllValuesNull()) {
-                    return BlockConverter.writeZeroes(out, vectorLength(block));
+                    return BlockConverter.writeZeroes(out, vectorByteSize(block));
                 }
 
                 // TODO could we "just" get the memory of the array and dump it?
-                int count = block.getPositionCount();
+                int count = BlockConverter.valueCount(block);
                 for (int i = 0; i < count; i++) {
                     out.writeDoubleLE(block.getDouble(i));
                 }
-                return vectorLength(block);
+                return (long) count * Double.BYTES;
             });
         }
 
-        private static int vectorLength(DoubleBlock b) {
-            return Double.BYTES * b.getPositionCount();
+        private static int vectorByteSize(DoubleBlock b) {
+            return Double.BYTES * BlockConverter.valueCount(b);
         }
     }
 
@@ -121,28 +125,31 @@ public AsInt32(String esqlType) {
         }
 
         @Override
-        public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
+        public void convert(Block b, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             IntBlock block = (IntBlock) b;
 
-            accumulateVectorValidity(bufs, bufWriters, block);
+            if (multivalued) {
+                addListOffsets(bufs, bufWriters, block);
+            }
+            accumulateVectorValidity(bufs, bufWriters, block, multivalued);
 
-            bufs.add(dummyArrowBuf(vectorLength(block)));
+            bufs.add(dummyArrowBuf(vectorByteSize(block)));
             bufWriters.add(out -> {
                 if (block.areAllValuesNull()) {
-                    return BlockConverter.writeZeroes(out, vectorLength(block));
+                    return BlockConverter.writeZeroes(out, vectorByteSize(block));
                 }
 
                 // TODO could we "just" get the memory of the array and dump it?
-                int count = block.getPositionCount();
+                int count = BlockConverter.valueCount(block);
                 for (int i = 0; i < count; i++) {
                     out.writeIntLE(block.getInt(i));
                 }
-                return vectorLength(block);
+                return (long) count * Integer.BYTES;
             });
         }
 
-        private static int vectorLength(IntBlock b) {
-            return Integer.BYTES * b.getPositionCount();
+        private static int vectorByteSize(Block b) {
+            return Integer.BYTES * BlockConverter.valueCount(b);
         }
     }
 
@@ -159,27 +166,31 @@ protected AsInt64(String esqlType, Types.MinorType minorType) {
         }
 
         @Override
-        public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
+        public void convert(Block b, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             LongBlock block = (LongBlock) b;
-            accumulateVectorValidity(bufs, bufWriters, block);
 
-            bufs.add(dummyArrowBuf(vectorLength(block)));
+            if (multivalued) {
+                addListOffsets(bufs, bufWriters, block);
+            }
+            accumulateVectorValidity(bufs, bufWriters, block, multivalued);
+
+            bufs.add(dummyArrowBuf(vectorByteSize(block)));
             bufWriters.add(out -> {
                 if (block.areAllValuesNull()) {
-                    return BlockConverter.writeZeroes(out, vectorLength(block));
+                    return BlockConverter.writeZeroes(out, vectorByteSize(block));
                 }
 
                 // TODO could we "just" get the memory of the array and dump it?
-                int count = block.getPositionCount();
+                int count = BlockConverter.valueCount(block);
                 for (int i = 0; i < count; i++) {
                     out.writeLongLE(block.getLong(i));
                 }
-                return vectorLength(block);
+                return (long) count * Long.BYTES;
             });
         }
 
-        private static int vectorLength(LongBlock b) {
-            return Long.BYTES * b.getPositionCount();
+        private static int vectorByteSize(LongBlock b) {
+            return Long.BYTES * BlockConverter.valueCount(b);
         }
     }
 
@@ -192,13 +203,17 @@ public AsBoolean(String esqlType) {
         }
 
         @Override
-        public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
+        public void convert(Block b, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             BooleanBlock block = (BooleanBlock) b;
-            accumulateVectorValidity(bufs, bufWriters, block);
 
-            bufs.add(dummyArrowBuf(vectorLength(block)));
+            if (multivalued) {
+                addListOffsets(bufs, bufWriters, block);
+            }
+            accumulateVectorValidity(bufs, bufWriters, block, multivalued);
+
+            bufs.add(dummyArrowBuf(vectorByteSize(block)));
             bufWriters.add(out -> {
-                int count = block.getPositionCount();
+                int count = BlockConverter.valueCount(block);
                 BitSet bits = new BitSet();
 
                 // Only set the bits that are true, writeBitSet will take
@@ -215,8 +230,8 @@ public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             });
         }
 
-        private static int vectorLength(BooleanBlock b) {
-            return BlockConverter.bitSetLength(b.getPositionCount());
+        private static int vectorByteSize(BooleanBlock b) {
+            return BlockConverter.bitSetLength(BlockConverter.valueCount(b));
         }
     }
 
@@ -230,27 +245,30 @@ public BytesRefConverter(String esqlType, Types.MinorType minorType) {
         }
 
         @Override
-        public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
+        public void convert(Block b, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             BytesRefBlock block = (BytesRefBlock) b;
 
-            BlockConverter.accumulateVectorValidity(bufs, bufWriters, block);
+            if (multivalued) {
+                addListOffsets(bufs, bufWriters, block);
+            }
+            accumulateVectorValidity(bufs, bufWriters, block, multivalued);
 
             // Offsets vector
-            bufs.add(dummyArrowBuf(offsetVectorLength(block)));
+            bufs.add(dummyArrowBuf(offsetvectorByteSize(block)));
 
             bufWriters.add(out -> {
                 if (block.areAllValuesNull()) {
-                    var count = block.getPositionCount() + 1;
+                    var count = valueCount(block) + 1;
                     for (int i = 0; i < count; i++) {
                         out.writeIntLE(0);
                     }
-                    return offsetVectorLength(block);
+                    return offsetvectorByteSize(block);
                 }
 
                 // TODO could we "just" get the memory of the array and dump it?
                 BytesRef scratch = new BytesRef();
                 int offset = 0;
-                for (int i = 0; i < block.getPositionCount(); i++) {
+                for (int i = 0; i < valueCount(block); i++) {
                     out.writeIntLE(offset);
                     // FIXME: add a ByteRefsVector.getLength(position): there are some cases
                     // where getBytesRef will allocate, which isn't needed here.
@@ -259,11 +277,11 @@ public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
                     offset += v.length;
                 }
                 out.writeIntLE(offset);
-                return offsetVectorLength(block);
+                return offsetvectorByteSize(block);
             });
 
             // Data vector
-            bufs.add(BlockConverter.dummyArrowBuf(dataVectorLength(block)));
+            bufs.add(BlockConverter.dummyArrowBuf(dataVectorByteSize(block)));
 
             bufWriters.add(out -> {
                 if (block.areAllValuesNull()) {
@@ -273,7 +291,7 @@ public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
                 // TODO could we "just" get the memory of the array and dump it?
                 BytesRef scratch = new BytesRef();
                 long length = 0;
-                for (int i = 0; i < block.getPositionCount(); i++) {
+                for (int i = 0; i < valueCount(block); i++) {
                     BytesRef v = block.getBytesRef(i, scratch);
 
                     out.write(v.bytes, v.offset, v.length);
@@ -283,11 +301,11 @@ public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             });
         }
 
-        private static int offsetVectorLength(BytesRefBlock block) {
-            return Integer.BYTES * (block.getPositionCount() + 1);
+        private static int offsetvectorByteSize(BytesRefBlock block) {
+            return Integer.BYTES * (valueCount(block) + 1);
         }
 
-        private int dataVectorLength(BytesRefBlock block) {
+        private int dataVectorByteSize(BytesRefBlock block) {
             if (block.areAllValuesNull()) {
                 return 0;
             }
@@ -296,7 +314,7 @@ private int dataVectorLength(BytesRefBlock block) {
 
             int length = 0;
             BytesRef scratch = new BytesRef();
-            for (int i = 0; i < block.getPositionCount(); i++) {
+            for (int i = 0; i < valueCount(block); i++) {
                 BytesRef v = block.getBytesRef(i, scratch);
                 length += v.length;
             }
@@ -323,10 +341,10 @@ public TransformedBytesRef(String esqlType, Types.MinorType minorType, BiFunctio
         }
 
         @Override
-        public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
+        public void convert(Block b, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             BytesRefBlock block = (BytesRefBlock) b;
             try (BytesRefBlock transformed = transformValues(block)) {
-                super.convert(transformed, bufs, bufWriters);
+                super.convert(transformed, multivalued, bufs, bufWriters);
             }
         }
 
@@ -336,20 +354,40 @@ public void convert(Block b, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
         private BytesRefBlock transformValues(BytesRefBlock block) {
             try (BytesRefBlock.Builder builder = block.blockFactory().newBytesRefBlockBuilder(block.getPositionCount())) {
                 BytesRef scratch = new BytesRef();
-                for (int i = 0; i < block.getPositionCount(); i++) {
-                    if (block.isNull(i)) {
-                        builder.appendNull();
-                    } else {
-                        BytesRef bytes = block.getBytesRef(i, scratch);
-                        if (bytes.length != 0) {
-                            bytes = valueConverter.apply(bytes, scratch);
+                if (block.mayHaveMultivaluedFields() == false) {
+                    for (int pos = 0; pos < valueCount(block); pos++) {
+                        if (block.isNull(pos)) {
+                            builder.appendNull();
+                        } else {
+                            convertAndAppend(builder, block, pos, scratch);
+                        }
+                    }
+                } else {
+                    for (int pos = 0; pos < block.getPositionCount(); pos++) {
+                        if (block.isNull(pos)) {
+                            builder.appendNull();
+                        } else {
+                            builder.beginPositionEntry();
+                            int startPos = block.getFirstValueIndex(pos);
+                            int lastPos = block.getFirstValueIndex(pos + 1);
+                            for (int valuePos = startPos; valuePos < lastPos; valuePos++) {
+                                convertAndAppend(builder, block, valuePos, scratch);
+                            }
+                            builder.endPositionEntry();
                         }
-                        builder.appendBytesRef(bytes);
                     }
                 }
                 return builder.build();
             }
         }
+
+        private void convertAndAppend(BytesRefBlock.Builder builder, BytesRefBlock block, int position, BytesRef scratch) {
+            BytesRef bytes = block.getBytesRef(position, scratch);
+            if (bytes.length != 0) {
+                bytes = valueConverter.apply(bytes, scratch);
+            }
+            builder.appendBytesRef(bytes);
+        }
     }
 
     public static class AsVarChar extends BytesRefConverter {
@@ -370,7 +408,7 @@ public AsNull(String esqlType) {
         }
 
         @Override
-        public void convert(Block block, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
+        public void convert(Block block, boolean multivalued, List<ArrowBuf> bufs, List<BufWriter> bufWriters) {
             // Null vector in arrow has no associated buffers
             // See https://arrow.apache.org/docs/format/Columnar.html#null-layout
         }
@@ -386,15 +424,38 @@ private static int bitSetLength(int totalValues) {
         return (totalValues + 7) / 8;
     }
 
-    private static void accumulateVectorValidity(List<ArrowBuf> bufs, List<BufWriter> bufWriters, Block b) {
-        bufs.add(dummyArrowBuf(bitSetLength(b.getPositionCount())));
+    /**
+     * Get the value count for a block. For single-valued blocks this is the same as the position count.
+     * For multivalued blocks, this is the flattened number of items.
+     */
+    static int valueCount(Block block) {
+        int result = block.getFirstValueIndex(block.getPositionCount());
+
+        // firstValueIndex is always zero for all-null blocks.
+        if (result == 0 && block.areAllValuesNull()) {
+            result = block.getPositionCount();
+        }
+
+        return result;
+    }
+
+    private static void accumulateVectorValidity(List<ArrowBuf> bufs, List<BufWriter> bufWriters, Block b, boolean multivalued) {
+        // If that block is in a multivalued-column, validities are output in the parent Arrow List buffer (values themselves
+        // do not contain nulls per docvalues limitations).
+        if (multivalued || b.mayHaveNulls() == false) {
+            // Arrow IPC allows a compact form for "all true" validities using an empty buffer.
+            bufs.add(dummyArrowBuf(0));
+            bufWriters.add(w -> 0);
+            return;
+        }
+
+        int valueCount = b.getPositionCount();
+        bufs.add(dummyArrowBuf(bitSetLength(valueCount)));
         bufWriters.add(out -> {
-            if (b.mayHaveNulls() == false) {
-                return writeAllTrueValidity(out, b.getPositionCount());
-            } else if (b.areAllValuesNull()) {
-                return writeAllFalseValidity(out, b.getPositionCount());
+            if (b.areAllValuesNull()) {
+                return writeAllFalseValidity(out, valueCount);
             } else {
-                return writeValidities(out, b);
+                return writeValidities(out, b, valueCount);
             }
         });
     }
@@ -420,10 +481,10 @@ private static long writeAllFalseValidity(RecyclerBytesStreamOutput out, int val
         return count;
     }
 
-    private static long writeValidities(RecyclerBytesStreamOutput out, Block block) {
-        int valueCount = block.getPositionCount();
+    private static long writeValidities(RecyclerBytesStreamOutput out, Block block, int valueCount) {
         BitSet bits = new BitSet(valueCount);
         for (int i = 0; i < block.getPositionCount(); i++) {
+            // isNull is value indices, not multi-value positions
             if (block.isNull(i) == false) {
                 bits.set(i);
             }
@@ -449,4 +510,29 @@ private static long writeZeroes(RecyclerBytesStreamOutput out, int byteCount) {
         }
         return byteCount;
     }
+
+    private static void addListOffsets(List<ArrowBuf> bufs, List<BufWriter> bufWriters, Block block) {
+        // Add validity buffer
+        accumulateVectorValidity(bufs, bufWriters, block, false);
+
+        // Add offsets buffer
+        int bufferLen = Integer.BYTES * (block.getPositionCount() + 1);
+
+        bufs.add(dummyArrowBuf(bufferLen));
+        bufWriters.add(out -> {
+            if (block.mayHaveMultivaluedFields()) {
+                // '<=' is intentional to write the end position of the last item
+                for (int i = 0; i <= block.getPositionCount(); i++) {
+                    // TODO could we get the block's firstValueIndexes and dump it?
+                    out.writeIntLE(block.getFirstValueIndex(i));
+                }
+            } else {
+                for (int i = 0; i <= block.getPositionCount(); i++) {
+                    out.writeIntLE(i);
+                }
+            }
+
+            return bufferLen;
+        });
+    }
 }
diff --git a/x-pack/plugin/esql/arrow/src/test/java/org/elasticsearch/xpack/esql/arrow/ArrowResponseTests.java b/x-pack/plugin/esql/arrow/src/test/java/org/elasticsearch/xpack/esql/arrow/ArrowResponseTests.java
index cf49b37db2805..b187e49554f8b 100644
--- a/x-pack/plugin/esql/arrow/src/test/java/org/elasticsearch/xpack/esql/arrow/ArrowResponseTests.java
+++ b/x-pack/plugin/esql/arrow/src/test/java/org/elasticsearch/xpack/esql/arrow/ArrowResponseTests.java
@@ -19,7 +19,10 @@
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.ipc.ArrowStreamReader;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
 import org.apache.arrow.vector.util.VectorSchemaRootAppender;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
@@ -34,7 +37,6 @@
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
-import org.elasticsearch.compute.data.IntVectorBlock;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.test.ESTestCase;
@@ -42,6 +44,8 @@
 import org.elasticsearch.xpack.versionfield.Version;
 import org.junit.AfterClass;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -75,6 +79,7 @@ public static void afterClass() throws Exception {
     // Value creation, getters for ESQL and Arrow
 
     static final ValueType INTEGER_VALUES = new ValueTypeImpl<IntBlock.Builder, IntBlock, IntVector>(
+        "integer",
         factory -> factory.newIntBlockBuilder(0),
         block -> block.appendInt(randomInt()),
         (block, i, scratch) -> block.getInt(i),
@@ -82,6 +87,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType LONG_VALUES = new ValueTypeImpl<LongBlock.Builder, LongBlock, BigIntVector>(
+        "long",
         factory -> factory.newLongBlockBuilder(0),
         block -> block.appendLong(randomLong()),
         (block, i, scratch) -> block.getLong(i),
@@ -89,6 +95,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType ULONG_VALUES = new ValueTypeImpl<LongBlock.Builder, LongBlock, UInt8Vector>(
+        "ulong",
         factory -> factory.newLongBlockBuilder(0),
         block -> block.appendLong(randomLong()),
         (block, i, scratch) -> block.getLong(i),
@@ -96,6 +103,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType DATE_VALUES = new ValueTypeImpl<LongBlock.Builder, LongBlock, TimeStampMilliVector>(
+        "date",
         factory -> factory.newLongBlockBuilder(0),
         block -> block.appendLong(randomLong()),
         (block, i, scratch) -> block.getLong(i),
@@ -103,6 +111,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType DOUBLE_VALUES = new ValueTypeImpl<DoubleBlock.Builder, DoubleBlock, Float8Vector>(
+        "double",
         factory -> factory.newDoubleBlockBuilder(0),
         block -> block.appendDouble(randomDouble()),
         (block, i, scratch) -> block.getDouble(i),
@@ -110,6 +119,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType BOOLEAN_VALUES = new ValueTypeImpl<BooleanBlock.Builder, BooleanBlock, BitVector>(
+        "boolean",
         factory -> factory.newBooleanBlockBuilder(0),
         block -> block.appendBoolean(randomBoolean()),
         (b, i, s) -> b.getBoolean(i),
@@ -117,21 +127,23 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType TEXT_VALUES = new ValueTypeImpl<BytesRefBlock.Builder, BytesRefBlock, VarCharVector>(
+        "text",
         factory -> factory.newBytesRefBlockBuilder(0),
-        block -> block.appendBytesRef(new BytesRef("🚀" + randomAlphaOfLengthBetween(1, 20))),
+        block -> block.appendBytesRef(new BytesRef(randomUnicodeOfLengthBetween(1, 20))),
         (b, i, s) -> b.getBytesRef(i, s).utf8ToString(),
         (v, i) -> new String(v.get(i), StandardCharsets.UTF_8)
     );
 
     static final ValueType SOURCE_VALUES = new ValueTypeImpl<BytesRefBlock.Builder, BytesRefBlock, VarCharVector>(
+        "source",
         factory -> factory.newBytesRefBlockBuilder(0),
-        // Use a constant value, conversion is tested separately
-        block -> block.appendBytesRef(new BytesRef("{\"foo\": 42}")),
+        block -> block.appendBytesRef(new BytesRef("{\"foo\": " + randomIntBetween(-42, 42) + "}")),
         (b, i, s) -> b.getBytesRef(i, s).utf8ToString(),
         (v, i) -> new String(v.get(i), StandardCharsets.UTF_8)
     );
 
     static final ValueType IP_VALUES = new ValueTypeImpl<BytesRefBlock.Builder, BytesRefBlock, VarBinaryVector>(
+        "ip",
         factory -> factory.newBytesRefBlockBuilder(0),
         block -> {
             byte[] addr = InetAddressPoint.encode(randomIp(randomBoolean()));
@@ -143,6 +155,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType BINARY_VALUES = new ValueTypeImpl<BytesRefBlock.Builder, BytesRefBlock, VarBinaryVector>(
+        "binary",
         factory -> factory.newBytesRefBlockBuilder(0),
         block -> block.appendBytesRef(new BytesRef(randomByteArrayOfLength(randomIntBetween(1, 100)))),
         BytesRefBlock::getBytesRef,
@@ -150,6 +163,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType VERSION_VALUES = new ValueTypeImpl<BytesRefBlock.Builder, BytesRefBlock, VarCharVector>(
+        "version",
         factory -> factory.newBytesRefBlockBuilder(0),
         block -> block.appendBytesRef(new Version(between(0, 100) + "." + between(0, 100) + "." + between(0, 100)).toBytesRef()),
         (b, i, s) -> new Version(b.getBytesRef(i, s)).toString(),
@@ -157,6 +171,7 @@ public static void afterClass() throws Exception {
     );
 
     static final ValueType NULL_VALUES = new ValueTypeImpl<Block.Builder, Block, FieldVector>(
+        "null",
         factory -> factory.newBytesRefBlockBuilder(0),
         Block.Builder::appendNull,
         (b, i, s) -> b.isNull(i) ? null : "non-null in block",
@@ -201,9 +216,10 @@ public void testTestHarness() {
         TestBlock emptyBlock = TestBlock.create(BLOCK_FACTORY, testColumn, Density.Empty, 7);
 
         // Test that density works as expected
-        assertTrue(denseBlock.block instanceof IntVectorBlock);
-        assertEquals("IntArrayBlock", sparseBlock.block.getClass().getSimpleName()); // non-public class
-        assertEquals("ConstantNullBlock", emptyBlock.block.getClass().getSimpleName());
+        assertFalse(denseBlock.block.mayHaveNulls());
+        assertTrue(sparseBlock.block.mayHaveNulls());
+        assertFalse(sparseBlock.block.areAllValuesNull());
+        assertTrue(emptyBlock.block.areAllValuesNull());
 
         // Test that values iterator scans all pages
         List<TestPage> pages = Stream.of(denseBlock, sparseBlock, emptyBlock).map(b -> new TestPage(List.of(b))).toList();
@@ -229,7 +245,7 @@ public void testTestHarness() {
      */
     public void testSingleColumn() throws IOException {
         for (var type : VALUE_TYPES.keySet()) {
-            TestColumn testColumn = new TestColumn("foo", type, VALUE_TYPES.get(type));
+            TestColumn testColumn = new TestColumn("foo", type, VALUE_TYPES.get(type), false);
             List<TestPage> pages = new ArrayList<>();
 
             for (var density : Density.values()) {
@@ -248,7 +264,7 @@ public void testSingleBlock() throws IOException {
         String type = "text";
         Density density = Density.Dense;
 
-        TestColumn testColumn = new TestColumn("foo", type, VALUE_TYPES.get(type));
+        TestColumn testColumn = new TestColumn("foo", type, VALUE_TYPES.get(type), false);
         List<TestPage> pages = new ArrayList<>();
 
         TestBlock testBlock = TestBlock.create(BLOCK_FACTORY, testColumn, density, 10);
@@ -261,44 +277,156 @@ public void testSingleBlock() throws IOException {
     }
 
     /**
-     * Test that multivalued arrays are rejected
+     * Test a multivalued field with fixed size values.
      */
-    public void testMultivaluedField() throws IOException {
+    public void testMultivaluedInteger() throws IOException {
         IntBlock.Builder builder = BLOCK_FACTORY.newIntBlockBuilder(0);
+        builder.beginPositionEntry();
         builder.appendInt(42);
+        builder.appendInt(43);
+        builder.endPositionEntry();
+
+        // The multivalue can be null, but a multivalue cannot contain nulls.
+        // Calling appendNull within a begin/endEntry causes consistency checks to fail in build()
+        // See also https://github.com/elastic/elasticsearch/issues/114324
         builder.appendNull();
+
         builder.beginPositionEntry();
         builder.appendInt(44);
         builder.appendInt(45);
         builder.endPositionEntry();
+
+        // single value
         builder.appendInt(46);
+
         IntBlock block = builder.build();
+        builder.close();
 
-        // Consistency check
+        // Consistency check.
+        // AbstractArrayBlock.assertInvariants does some of these consistency checks, but those below
+        // specifically verify the assumptions on which the conversion to Arrow is built.
         assertTrue(block.mayHaveMultivaluedFields());
+        assertEquals(4, block.getPositionCount()); // counts null entries
+        assertEquals(5, block.getTotalValueCount()); // nulls aren't counted
+
+        // Value 0
+        assertEquals(2, block.getValueCount(0));
         assertEquals(0, block.getFirstValueIndex(0));
-        assertEquals(1, block.getValueCount(0));
+        assertEquals(42, block.getInt(block.getFirstValueIndex(0)));
+        assertEquals(43, block.getInt(block.getFirstValueIndex(0) + 1));
 
-        // null values still use one position in the array
+        // Value 1
         assertEquals(0, block.getValueCount(1));
-        assertEquals(1, block.getFirstValueIndex(1));
-        assertTrue(block.isNull(1));
-        assertEquals(0, block.getInt(1));
+        assertTrue(block.isNull(1)); // This is the position index, not value index
+        // No value, but still occupies a value slot with zero
+        assertEquals(2, block.getFirstValueIndex(1));
+        assertEquals(0, block.getInt(block.getFirstValueIndex(1)));
+        assertEquals(3, block.getFirstValueIndex(2));
 
-        assertEquals(2, block.getFirstValueIndex(2));
+        // Value 2
         assertEquals(2, block.getValueCount(2));
-        assertEquals(2, block.getFirstValueIndex(2));
+        assertEquals(3, block.getFirstValueIndex(2));
+        assertEquals(44, block.getInt(block.getFirstValueIndex(2)));
         assertEquals(45, block.getInt(block.getFirstValueIndex(2) + 1));
 
-        assertEquals(4, block.getFirstValueIndex(3));
+        // Value 3
+        assertEquals(1, block.getValueCount(3));
+        assertEquals(5, block.getFirstValueIndex(3));
+        assertEquals(46, block.getInt(block.getFirstValueIndex(3)));
 
-        var column = TestColumn.create("some-field", "integer");
-        TestCase testCase = new TestCase(List.of(column), List.of(new TestPage(List.of(new TestBlock(column, block, Density.Dense)))));
+        // End of block
+        assertEquals(6, block.getFirstValueIndex(4));
 
-        IllegalArgumentException exc = assertThrows(IllegalArgumentException.class, () -> compareEsqlAndArrow(testCase));
+        var column = TestColumn.create("some-field", "integer", true);
+        TestCase testCase = new TestCase(List.of(column), List.of(new TestPage(List.of(TestBlock.create(column, block)))));
 
-        assertEquals("ES|QL response field [some-field] is multi-valued. This isn't supported yet by the Arrow format", exc.getMessage());
+        compareEsqlAndArrow(testCase);
+    }
+
+    /**
+     * Test a multivalued field with variable size values.
+     */
+    public void testMultivalueString() throws IOException {
+        BytesRefBlock.Builder builder = BLOCK_FACTORY.newBytesRefBlockBuilder(0);
+
+        builder.beginPositionEntry();
+        builder.appendBytesRef(new BytesRef("a"));
+        builder.appendBytesRef(new BytesRef("b"));
+        builder.endPositionEntry();
 
+        builder.beginPositionEntry();
+        builder.appendBytesRef(new BytesRef("c"));
+        builder.appendBytesRef(new BytesRef("d"));
+        builder.endPositionEntry();
+
+        BytesRefBlock block = builder.build();
+        builder.close();
+
+        var column = TestColumn.create("some-field", "text");
+        TestCase testCase = new TestCase(List.of(column), List.of(new TestPage(List.of(TestBlock.create(column, block)))));
+
+        compareEsqlAndArrow(testCase);
+    }
+
+    // Test exercising Arrow's multivalue API
+    public void testMultiValueArrow() throws IOException {
+
+        byte[] bytes;
+
+        try (ListVector listVector = ListVector.empty("some-field", ALLOCATOR)) {
+            UnionListWriter writer = listVector.getWriter();
+
+            writer.startList();
+            writer.writeInt(42); // 0x2A
+            writer.writeInt(43); // 0x2A
+            writer.endList();
+
+            writer.startList();
+            // Size is zero without a writeNull()
+            writer.writeNull(); // Adds a null value in that list
+            writer.endList();
+
+            writer.startList();
+            writer.writeInt(44); // 0x2C
+            writer.writeInt(45); // 0x2D
+            writer.endList();
+
+            writer.startList();
+            writer.writeInt(46); // 0x2E
+            writer.endList();
+
+            listVector.setValueCount(4);
+            bytes = getBytes(listVector);
+        }
+
+        try (var reader = new ArrowStreamReader(new ByteArrayInputStream(bytes), ALLOCATOR)) {
+            var root = reader.getVectorSchemaRoot();
+            reader.loadNextBatch();
+
+            ListVector listVector = (ListVector) root.getVector("some-field");
+
+            assertEquals(4, listVector.getValueCount());
+            assertEquals(List.of(42, 43), listVector.getObject(0));
+            assertEquals(Collections.singletonList((Integer) null), listVector.getObject(1));
+            assertEquals(List.of(44, 45), listVector.getObject(2));
+            assertEquals(List.of(46), listVector.getObject(3));
+        }
+    }
+
+    private static byte[] getBytes(ListVector listVector) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        var fields = List.of(listVector.getField());
+        List<FieldVector> vectors = List.of(listVector);
+
+        try (
+            VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors);
+            ArrowStreamWriter arrowWriter = new ArrowStreamWriter(root, null, baos);
+        ) {
+            arrowWriter.start();
+            arrowWriter.writeBatch();
+            arrowWriter.end();
+        }
+        return baos.toByteArray();
     }
 
     /**
@@ -319,10 +447,6 @@ public void testRandomTypesAndSize() throws IOException {
             .toList();
 
         TestCase testCase = new TestCase(columns, pages);
-        // System.out.println(testCase);
-        // for (TestPage page: pages) {
-        // System.out.println(page);
-        // }
 
         compareEsqlAndArrow(testCase);
     }
@@ -347,8 +471,13 @@ private void compareEsqlAndArrow(TestCase testCase, VectorSchemaRoot root) {
             var esqlValuesIterator = new EsqlValuesIterator(testCase, i);
             var arrowValuesIterator = new ArrowValuesIterator(testCase, root, i);
 
+            int line = 0;
+
             while (esqlValuesIterator.hasNext() && arrowValuesIterator.hasNext()) {
-                assertEquals(esqlValuesIterator.next(), arrowValuesIterator.next());
+                Object esqlValue = esqlValuesIterator.next();
+                Object arrowValue = arrowValuesIterator.next();
+                assertEquals(("line " + line), esqlValue, arrowValue);
+                line++;
             }
 
             // Make sure we entirely consumed both sides.
@@ -387,7 +516,6 @@ private VectorSchemaRoot toArrowVectors(TestCase testCase) throws IOException {
     static class EsqlValuesIterator implements Iterator<Object> {
         private final int fieldPos;
         private final ValueType type;
-        private final BytesRef scratch = new BytesRef();
         private final Iterator<TestPage> pages;
 
         private TestPage page;
@@ -412,7 +540,7 @@ public Object next() {
                 throw new NoSuchElementException();
             }
             Block block = page.blocks.get(fieldPos).block;
-            Object result = block.isNull(position) ? null : type.valueAt(block, position, scratch);
+            Object result = block.isNull(position) ? null : type.valueAt(block, position, new BytesRef());
             position++;
             if (position >= block.getPositionCount()) {
                 position = 0;
@@ -475,9 +603,13 @@ public String toString() {
         }
     }
 
-    record TestColumn(String name, String type, ValueType valueType) {
+    record TestColumn(String name, String type, ValueType valueType, boolean multivalue) {
         static TestColumn create(String name, String type) {
-            return new TestColumn(name, type, VALUE_TYPES.get(type));
+            return create(name, type, randomBoolean());
+        }
+
+        static TestColumn create(String name, String type, boolean multivalue) {
+            return new TestColumn(name, type, VALUE_TYPES.get(type), multivalue);
         }
     }
 
@@ -498,6 +630,18 @@ public String toString() {
 
     record TestBlock(TestColumn column, Block block, Density density) {
 
+        static TestBlock create(TestColumn column, Block block) {
+            Density density;
+            if (block.areAllValuesNull()) {
+                density = Density.Empty;
+            } else if (block.mayHaveNulls()) {
+                density = Density.Sparse;
+            } else {
+                density = Density.Dense;
+            }
+            return new TestBlock(column, block, density);
+        }
+
         static TestBlock create(BlockFactory factory, TestColumn column, int positions) {
             return create(factory, column, randomFrom(Density.values()), positions);
         }
@@ -517,10 +661,21 @@ static TestBlock create(BlockFactory factory, TestColumn column, Density density
                     start = 2;
                 }
                 for (int i = start; i < positions; i++) {
-                    valueType.addValue(builder, density);
+                    // If multivalued, randomly insert a series of values if the type isn't null (nulls are not allowed in multivalues)
+                    if (column.multivalue && column.valueType != NULL_VALUES && randomBoolean()) {
+                        builder.beginPositionEntry();
+                        int numEntries = randomIntBetween(2, 5);
+                        for (int j = 0; j < numEntries; j++) {
+                            valueType.addValue(builder, Density.Dense);
+                        }
+                        builder.endPositionEntry();
+                    } else {
+                        valueType.addValue(builder, density);
+                    }
                 }
                 // Will create an ArrayBlock if there are null values, VectorBlock otherwise
                 block = builder.build();
+                assertEquals(positions, block.getPositionCount());
             }
             return new TestBlock(column, block, density);
         }
@@ -553,17 +708,20 @@ interface ValueType {
     public static class ValueTypeImpl<BlockBT extends Block.Builder, BlockT extends Block, VectorT extends ValueVector>
         implements
             ValueType {
+        private final String name;
         private final Function<BlockFactory, BlockBT> builderCreator;
         private final Consumer<BlockBT> valueAdder;
         private final TriFunction<BlockT, Integer, BytesRef, Object> blockGetter;
         private final BiFunction<VectorT, Integer, Object> vectorGetter;
 
         public ValueTypeImpl(
+            String name,
             Function<BlockFactory, BlockBT> builderCreator,
             Consumer<BlockBT> valueAdder,
             TriFunction<BlockT, Integer, BytesRef, Object> blockGetter,
             BiFunction<VectorT, Integer, Object> vectorGetter
         ) {
+            this.name = name;
             this.builderCreator = builderCreator;
             this.valueAdder = valueAdder;
             this.blockGetter = blockGetter;
@@ -588,13 +746,35 @@ public void addValue(Block.Builder builder, Density density) {
         @Override
         @SuppressWarnings("unchecked")
         public Object valueAt(Block block, int position, BytesRef scratch) {
-            return blockGetter.apply((BlockT) block, position, scratch);
+            // Build the list of values
+            var values = new ArrayList<>();
+            for (int i = block.getFirstValueIndex(position); i < block.getFirstValueIndex(position + 1); i++) {
+                values.add(blockGetter.apply((BlockT) block, i, scratch));
+                scratch = new BytesRef(); // do not overwrite previous value
+            }
+            return values.size() == 1 ? values.getFirst() : values;
         }
 
         @Override
         @SuppressWarnings("unchecked")
         public Object valueAt(ValueVector arrowVec, int position) {
-            return vectorGetter.apply((VectorT) arrowVec, position);
+            if (arrowVec instanceof ListVector listVector) {
+                var type = listVector.getField().getMetadata().get("elastic:type");
+                // Build the list of values
+                var valueVec = listVector.getDataVector();
+                var values = new ArrayList<>();
+                for (int i = listVector.getElementStartIndex(position); i < listVector.getElementEndIndex(position); i++) {
+                    values.add(vectorGetter.apply((VectorT) valueVec, i));
+                }
+                return values.size() == 1 ? values.getFirst() : values;
+            } else {
+                return vectorGetter.apply((VectorT) arrowVec, position);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return name;
         }
     }
 }
diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index c8d704cd2b8bf..766d0c0f13892 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -1,6 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask;
 import org.elasticsearch.gradle.internal.util.SourceDirectoryCommandLineArgumentProvider;
+import static org.elasticsearch.gradle.util.PlatformUtils.normalize
 
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
@@ -56,7 +57,7 @@ def generatedSourceDir = projectDirectory.dir("src/main/generated")
 tasks.named("compileJava").configure {
   options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(generatedSourceDir))
   // IntelliJ sticks generated files here and we can't stop it....
-  exclude { it.file.toString().contains("src/main/generated-src/generated") }
+  exclude { normalize(it.file.toString()).contains("src/main/generated-src/generated") }
 }
 
 interface Injected {
@@ -262,8 +263,8 @@ tasks.register("regen") {
 tasks.named("spotlessJava") { dependsOn stringTemplates }
 tasks.named('checkstyleMain').configure {
   excludes = [ "**/*.java.st" ]
-  exclude { it.file.toString().contains("src/main/generated-src/generated") }
-  exclude { it.file.toString().contains("src/main/generated") }
+  exclude { normalize(it.file.toString()).contains("src/main/generated-src/generated") }
+  exclude { normalize(it.file.toString()).contains("src/main/generated") }
 }
 
 def prop(Type, type, TYPE, BYTES, Array) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java
index 7fcb412de5e2b..b8ea7658a8247 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.compute.aggregation.blockhash;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.BitArray;
-import org.elasticsearch.common.util.BytesRefArray;
 import org.elasticsearch.common.util.BytesRefHash;
 import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction;
 import org.elasticsearch.compute.aggregation.SeenGroupIds;
@@ -23,15 +20,16 @@
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.OrdinalBytesRefBlock;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeBytesRef;
+import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeInt;
 import org.elasticsearch.core.ReleasableIterator;
 
-import java.io.IOException;
-
 /**
  * Maps a {@link BytesRefBlock} column to group ids.
+ * This class is generated. Do not edit it.
  */
 final class BytesRefBlockHash extends BlockHash {
     private final int channel;
@@ -54,6 +52,7 @@ final class BytesRefBlockHash extends BlockHash {
 
     @Override
     public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
+        // TODO track raw counts and which implementation we pick for the profiler - #114008
         var block = page.getBlock(channel);
         if (block.areAllValuesNull()) {
             seenNull = true;
@@ -76,6 +75,10 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
     }
 
     IntVector add(BytesRefVector vector) {
+        var ordinals = vector.asOrdinals();
+        if (ordinals != null) {
+            return addOrdinalsVector(ordinals);
+        }
         BytesRef scratch = new BytesRef();
         int positions = vector.getPositionCount();
         try (var builder = blockFactory.newIntVectorFixedBuilder(positions)) {
@@ -113,15 +116,29 @@ public ReleasableIterator<IntBlock> lookup(Page page, ByteSizeValue targetBlockS
         return ReleasableIterator.single(lookup(vector));
     }
 
-    private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) {
-        var inputOrds = inputBlock.getOrdinalsBlock();
+    private IntVector addOrdinalsVector(OrdinalBytesRefVector inputBlock) {
+        IntVector inputOrds = inputBlock.getOrdinalsVector();
         try (
-            var builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount());
+            var builder = blockFactory.newIntVectorBuilder(inputOrds.getPositionCount());
             var hashOrds = add(inputBlock.getDictionaryVector())
         ) {
-            for (int i = 0; i < inputOrds.getPositionCount(); i++) {
-                int valueCount = inputOrds.getValueCount(i);
-                int firstIndex = inputOrds.getFirstValueIndex(i);
+            for (int p = 0; p < inputOrds.getPositionCount(); p++) {
+                int ord = hashOrds.getInt(inputOrds.getInt(p));
+                builder.appendInt(ord);
+            }
+            return builder.build();
+        }
+    }
+
+    private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) {
+        try (
+            IntBlock inputOrds = new MultivalueDedupeInt(inputBlock.getOrdinalsBlock()).dedupeToBlockAdaptive(blockFactory);
+            IntBlock.Builder builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount());
+            IntVector hashOrds = add(inputBlock.getDictionaryVector())
+        ) {
+            for (int p = 0; p < inputOrds.getPositionCount(); p++) {
+                int valueCount = inputOrds.getValueCount(p);
+                int firstIndex = inputOrds.getFirstValueIndex(p);
                 switch (valueCount) {
                     case 0 -> {
                         builder.appendInt(0);
@@ -132,9 +149,11 @@ private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) {
                         builder.appendInt(ord);
                     }
                     default -> {
+                        int start = firstIndex;
+                        int end = firstIndex + valueCount;
                         builder.beginPositionEntry();
-                        for (int v = 0; v < valueCount; v++) {
-                            int ord = hashOrds.getInt(inputOrds.getInt(firstIndex + i));
+                        for (int i = start; i < end; i++) {
+                            int ord = hashOrds.getInt(inputOrds.getInt(i));
                             builder.appendInt(ord);
                         }
                         builder.endPositionEntry();
@@ -173,26 +192,21 @@ public BytesRefBlock[] getKeys() {
          * without and still read from the block.
          */
         // TODO replace with takeBytesRefsOwnership ?!
+        final BytesRef spare = new BytesRef();
         if (seenNull) {
             try (var builder = blockFactory.newBytesRefBlockBuilder(Math.toIntExact(hash.size() + 1))) {
                 builder.appendNull();
-                BytesRef spare = new BytesRef();
                 for (long i = 0; i < hash.size(); i++) {
                     builder.appendBytesRef(hash.get(i, spare));
                 }
                 return new BytesRefBlock[] { builder.build() };
             }
         }
-
-        final int size = Math.toIntExact(hash.size());
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            hash.getBytesRefs().writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                return new BytesRefBlock[] {
-                    blockFactory.newBytesRefArrayVector(new BytesRefArray(in, BigArrays.NON_RECYCLING_INSTANCE), size).asBlock() };
+        try (var builder = blockFactory.newBytesRefBlockBuilder(Math.toIntExact(hash.size()))) {
+            for (long i = 0; i < hash.size(); i++) {
+                builder.appendBytesRef(hash.get(i, spare));
             }
-        } catch (IOException e) {
-            throw new IllegalStateException(e);
+            return new BytesRefBlock[] { builder.build() };
         }
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java
index bd9d752302ae3..c9c672112a630 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java
@@ -28,6 +28,7 @@
 
 /**
  * Maps a {@link DoubleBlock} column to group ids.
+ * This class is generated. Do not edit it.
  */
 final class DoubleBlockHash extends BlockHash {
     private final int channel;
@@ -50,6 +51,7 @@ final class DoubleBlockHash extends BlockHash {
 
     @Override
     public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
+        // TODO track raw counts and which implementation we pick for the profiler - #114008
         var block = page.getBlock(channel);
         if (block.areAllValuesNull()) {
             seenNull = true;
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java
index 5b1b48bd270ab..13b60c6f1fec5 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java
@@ -26,6 +26,7 @@
 
 /**
  * Maps a {@link IntBlock} column to group ids.
+ * This class is generated. Do not edit it.
  */
 final class IntBlockHash extends BlockHash {
     private final int channel;
@@ -48,6 +49,7 @@ final class IntBlockHash extends BlockHash {
 
     @Override
     public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
+        // TODO track raw counts and which implementation we pick for the profiler - #114008
         var block = page.getBlock(channel);
         if (block.areAllValuesNull()) {
             seenNull = true;
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java
index 074ccb2f7cd7d..5252bd742ec51 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java
@@ -28,6 +28,7 @@
 
 /**
  * Maps a {@link LongBlock} column to group ids.
+ * This class is generated. Do not edit it.
  */
 final class LongBlockHash extends BlockHash {
     private final int channel;
@@ -50,6 +51,7 @@ final class LongBlockHash extends BlockHash {
 
     @Override
     public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
+        // TODO track raw counts and which implementation we pick for the profiler - #114008
         var block = page.getBlock(channel);
         if (block.areAllValuesNull()) {
             seenNull = true;
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunction.java
index 3e38b6d6fe9fa..8d3dbf3164c47 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunction.java
@@ -97,7 +97,7 @@ public void addIntermediateInput(int positionOffset, IntVector groupIdVector, Pa
 
     @Override
     public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) {
-        next.addIntermediateRowInput(groupId, input, position);
+        next.addIntermediateRowInput(groupId, ((FilteredGroupingAggregatorFunction) input).next(), position);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java
index fe1a07e8e16a6..919cb92f79260 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.BitArray;
 import org.elasticsearch.common.util.BytesRefHash;
+import org.elasticsearch.common.util.Int3Hash;
 import org.elasticsearch.common.util.LongHash;
 import org.elasticsearch.common.util.LongLongHash;
 import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction;
@@ -28,14 +29,37 @@
 import java.util.List;
 
 /**
- * A specialized hash table implementation maps values of a {@link Block} to ids (in longs).
- * This class delegates to {@link LongHash} or {@link BytesRefHash}.
- *
- * @see LongHash
- * @see BytesRefHash
+ * Specialized hash table implementations that map rows to a <strong>set</strong>
+ * of bucket IDs to which they belong to implement {@code GROUP BY} expressions.
+ * <p>
+ *     A row is always in at least one bucket so the results are never {@code null}.
+ *     {@code null} valued key columns will map to some integer bucket id.
+ *     If none of key columns are multivalued then the output is always an
+ *     {@link IntVector}. If any of the key are multivalued then a row is
+ *     in a bucket for each value. If more than one key is multivalued then
+ *     the row is in the combinatorial explosion of all value combinations.
+ *     Luckily for the number of values rows can only be in each bucket once.
+ *     Unluckily, it's the responsibility of {@link BlockHash} to remove those
+ *     duplicates.
+ * </p>
+ * <p>
+ *     These classes typically delegate to some combination of {@link BytesRefHash},
+ *     {@link LongHash}, {@link LongLongHash}, {@link Int3Hash}. They don't
+ *     <strong>technically</strong> have to be hash tables, so long as they
+ *     implement the deduplication semantics above and vend integer ids.
+ * </p>
+ * <p>
+ *     The integer ids are assigned to offsets into arrays of aggregation states
+ *     so its permissible to have gaps in the ints. But large gaps are a bad
+ *     idea because they'll waste space in the aggregations that use these
+ *     positions. For example, {@link BooleanBlockHash} assigns {@code 0} to
+ *     {@code null}, {@code 1} to {@code false}, and {@code 1} to {@code true}
+ *     and that's <strong>fine</strong> and simple and good because it'll never
+ *     leave a big gap, even if we never see {@code null}.
+ * </p>
  */
 public abstract sealed class BlockHash implements Releasable, SeenGroupIds //
-    permits BooleanBlockHash, BytesRefBlockHash, DoubleBlockHash, IntBlockHash, LongBlockHash, BytesRef3BlockHash, //
+    permits BooleanBlockHash, BytesRefBlockHash, DoubleBlockHash, IntBlockHash, LongBlockHash, BytesRef2BlockHash, BytesRef3BlockHash, //
     NullBlockHash, PackedValuesBlockHash, BytesRefLongBlockHash, LongLongBlockHash, TimeSeriesBlockHash {
 
     protected final BlockFactory blockFactory;
@@ -98,8 +122,19 @@ public static BlockHash build(List<GroupSpec> groups, BlockFactory blockFactory,
         if (groups.size() == 1) {
             return newForElementType(groups.get(0).channel(), groups.get(0).elementType(), blockFactory);
         }
-        if (groups.size() == 3 && groups.stream().allMatch(g -> g.elementType == ElementType.BYTES_REF)) {
-            return new BytesRef3BlockHash(blockFactory, groups.get(0).channel, groups.get(1).channel, groups.get(2).channel, emitBatchSize);
+        if (groups.stream().allMatch(g -> g.elementType == ElementType.BYTES_REF)) {
+            switch (groups.size()) {
+                case 2:
+                    return new BytesRef2BlockHash(blockFactory, groups.get(0).channel, groups.get(1).channel, emitBatchSize);
+                case 3:
+                    return new BytesRef3BlockHash(
+                        blockFactory,
+                        groups.get(0).channel,
+                        groups.get(1).channel,
+                        groups.get(2).channel,
+                        emitBatchSize
+                    );
+            }
         }
         if (allowBrokenOptimizations && groups.size() == 2) {
             var g1 = groups.get(0);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java
index 4c2817588904a..ecbf292b077ea 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java
@@ -25,8 +25,9 @@
 import static org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeBoolean.TRUE_ORD;
 
 /**
- * Maps a {@link BooleanBlock} column to group ids. Assigns group
- * {@code 0} to {@code false} and group {@code 1} to {@code true}.
+ * Maps a {@link BooleanBlock} column to group ids. Assigns
+ * {@code 0} to {@code null}, {@code 1} to {@code false}, and
+ * {@code 2} to {@code true}.
  */
 final class BooleanBlockHash extends BlockHash {
     private final int channel;
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef2BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef2BlockHash.java
new file mode 100644
index 0000000000000..ff25aa1381004
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef2BlockHash.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation.blockhash;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.BitArray;
+import org.elasticsearch.common.util.LongHash;
+import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.ReleasableIterator;
+import org.elasticsearch.core.Releasables;
+
+import java.util.Locale;
+
+/**
+ * Maps two {@link BytesRefBlock}s to group ids.
+ */
+final class BytesRef2BlockHash extends BlockHash {
+    private final int emitBatchSize;
+    private final int channel1;
+    private final int channel2;
+    private final BytesRefBlockHash hash1;
+    private final BytesRefBlockHash hash2;
+    private final LongHash finalHash;
+
+    BytesRef2BlockHash(BlockFactory blockFactory, int channel1, int channel2, int emitBatchSize) {
+        super(blockFactory);
+        this.emitBatchSize = emitBatchSize;
+        this.channel1 = channel1;
+        this.channel2 = channel2;
+        boolean success = false;
+        try {
+            this.hash1 = new BytesRefBlockHash(channel1, blockFactory);
+            this.hash2 = new BytesRefBlockHash(channel2, blockFactory);
+            this.finalHash = new LongHash(1, blockFactory.bigArrays());
+            success = true;
+        } finally {
+            if (success == false) {
+                close();
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(hash1, hash2, finalHash);
+    }
+
+    @Override
+    public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
+        BytesRefBlock b1 = page.getBlock(channel1);
+        BytesRefBlock b2 = page.getBlock(channel2);
+        BytesRefVector v1 = b1.asVector();
+        BytesRefVector v2 = b2.asVector();
+        if (v1 != null && v2 != null) {
+            addVectors(v1, v2, addInput);
+        } else {
+            try (IntBlock k1 = hash1.add(b1); IntBlock k2 = hash2.add(b2)) {
+                try (AddWork work = new AddWork(k1, k2, addInput)) {
+                    work.add();
+                }
+            }
+        }
+    }
+
+    private void addVectors(BytesRefVector v1, BytesRefVector v2, GroupingAggregatorFunction.AddInput addInput) {
+        final int positionCount = v1.getPositionCount();
+        try (IntVector.FixedBuilder ordsBuilder = blockFactory.newIntVectorFixedBuilder(positionCount)) {
+            try (IntVector k1 = hash1.add(v1); IntVector k2 = hash2.add(v2)) {
+                for (int p = 0; p < positionCount; p++) {
+                    long ord = ord(k1.getInt(p), k2.getInt(p));
+                    ordsBuilder.appendInt(p, Math.toIntExact(ord));
+                }
+            }
+            try (IntVector ords = ordsBuilder.build()) {
+                addInput.add(0, ords);
+            }
+        }
+    }
+
+    private class AddWork extends AddPage {
+        final IntBlock b1;
+        final IntBlock b2;
+
+        AddWork(IntBlock b1, IntBlock b2, GroupingAggregatorFunction.AddInput addInput) {
+            super(blockFactory, emitBatchSize, addInput);
+            this.b1 = b1;
+            this.b2 = b2;
+        }
+
+        void add() {
+            int positionCount = b1.getPositionCount();
+            for (int i = 0; i < positionCount; i++) {
+                int v1 = b1.getValueCount(i);
+                int v2 = b2.getValueCount(i);
+                int first1 = b1.getFirstValueIndex(i);
+                int first2 = b2.getFirstValueIndex(i);
+                if (v1 == 1 && v2 == 1) {
+                    long ord = ord(b1.getInt(first1), b2.getInt(first2));
+                    appendOrdSv(i, Math.toIntExact(ord));
+                    continue;
+                }
+                for (int i1 = 0; i1 < v1; i1++) {
+                    int k1 = b1.getInt(first1 + i1);
+                    for (int i2 = 0; i2 < v2; i2++) {
+                        int k2 = b2.getInt(first2 + i2);
+                        long ord = ord(k1, k2);
+                        appendOrdInMv(i, Math.toIntExact(ord));
+                    }
+                }
+                finishMv();
+            }
+            flushRemaining();
+        }
+    }
+
+    private long ord(int k1, int k2) {
+        return hashOrdToGroup(finalHash.add((long) k2 << 32 | k1));
+    }
+
+    @Override
+    public ReleasableIterator<IntBlock> lookup(Page page, ByteSizeValue targetBlockSize) {
+        throw new UnsupportedOperationException("TODO");
+    }
+
+    @Override
+    public Block[] getKeys() {
+        // TODO Build Ordinals blocks #114010
+        final int positions = (int) finalHash.size();
+        final BytesRef scratch = new BytesRef();
+        final BytesRefBlock[] outputBlocks = new BytesRefBlock[2];
+        try {
+            try (BytesRefBlock.Builder b1 = blockFactory.newBytesRefBlockBuilder(positions)) {
+                for (int i = 0; i < positions; i++) {
+                    int k1 = (int) (finalHash.get(i) & 0xffffL);
+                    if (k1 == 0) {
+                        b1.appendNull();
+                    } else {
+                        b1.appendBytesRef(hash1.hash.get(k1 - 1, scratch));
+                    }
+                }
+                outputBlocks[0] = b1.build();
+            }
+            try (BytesRefBlock.Builder b2 = blockFactory.newBytesRefBlockBuilder(positions)) {
+                for (int i = 0; i < positions; i++) {
+                    int k2 = (int) (finalHash.get(i) >>> 32);
+                    if (k2 == 0) {
+                        b2.appendNull();
+                    } else {
+                        b2.appendBytesRef(hash2.hash.get(k2 - 1, scratch));
+                    }
+                }
+                outputBlocks[1] = b2.build();
+            }
+            return outputBlocks;
+        } finally {
+            if (outputBlocks[outputBlocks.length - 1] == null) {
+                Releasables.close(outputBlocks);
+            }
+        }
+    }
+
+    @Override
+    public BitArray seenGroupIds(BigArrays bigArrays) {
+        return new Range(0, Math.toIntExact(finalHash.size())).seenGroupIds(bigArrays);
+    }
+
+    @Override
+    public IntVector nonEmpty() {
+        return IntVector.range(0, Math.toIntExact(finalHash.size()), blockFactory);
+    }
+
+    @Override
+    public String toString() {
+        return String.format(
+            Locale.ROOT,
+            "BytesRef2BlockHash{keys=[channel1=%d, channel2=%d], entries=%d}",
+            channel1,
+            channel2,
+            finalHash.size()
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java
index 54bf068b4de33..987a808ed7950 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java
@@ -85,7 +85,6 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
     private void addVectors(BytesRefVector v1, BytesRefVector v2, BytesRefVector v3, GroupingAggregatorFunction.AddInput addInput) {
         final int positionCount = v1.getPositionCount();
         try (IntVector.FixedBuilder ordsBuilder = blockFactory.newIntVectorFixedBuilder(positionCount)) {
-            // TODO: enable ordinal vectors in BytesRefBlockHash
             try (IntVector k1 = hash1.add(v1); IntVector k2 = hash2.add(v2); IntVector k3 = hash3.add(v3)) {
                 for (int p = 0; p < positionCount; p++) {
                     long ord = hashOrdToGroup(finalHash.add(k1.getInt(p), k2.getInt(p), k3.getInt(p)));
@@ -148,6 +147,7 @@ public ReleasableIterator<IntBlock> lookup(Page page, ByteSizeValue targetBlockS
 
     @Override
     public Block[] getKeys() {
+        // TODO Build Ordinals blocks #114010
         final int positions = (int) finalHash.size();
         final BytesRef scratch = new BytesRef();
         final BytesRefBlock[] outputBlocks = new BytesRefBlock[3];
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st
index b4f700980558e..2a3d1143236ac 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st
@@ -9,15 +9,10 @@ package org.elasticsearch.compute.aggregation.blockhash;
 
 $if(BytesRef)$
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
 $endif$
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.BitArray;
-$if(BytesRef)$
-import org.elasticsearch.common.util.BytesRefArray;
-$endif$
 import org.elasticsearch.common.util.$Hash$;
 import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction;
 import org.elasticsearch.compute.aggregation.SeenGroupIds;
@@ -28,6 +23,7 @@ import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.OrdinalBytesRefBlock;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 $elseif(double)$
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -51,17 +47,19 @@ $endif$
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe$Type$;
+$if(BytesRef)$
+import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeInt;
+$endif$
 import org.elasticsearch.core.ReleasableIterator;
 
 $if(BytesRef)$
-import java.io.IOException;
-
 $else$
 import java.util.BitSet;
 
 $endif$
 /**
  * Maps a {@link $Type$Block} column to group ids.
+ * This class is generated. Do not edit it.
  */
 final class $Type$BlockHash extends BlockHash {
     private final int channel;
@@ -84,6 +82,7 @@ final class $Type$BlockHash extends BlockHash {
 
     @Override
     public void add(Page page, GroupingAggregatorFunction.AddInput addInput) {
+        // TODO track raw counts and which implementation we pick for the profiler - #114008
         var block = page.getBlock(channel);
         if (block.areAllValuesNull()) {
             seenNull = true;
@@ -107,6 +106,10 @@ final class $Type$BlockHash extends BlockHash {
 
     IntVector add($Type$Vector vector) {
 $if(BytesRef)$
+        var ordinals = vector.asOrdinals();
+        if (ordinals != null) {
+            return addOrdinalsVector(ordinals);
+        }
         BytesRef scratch = new BytesRef();
 $endif$
         int positions = vector.getPositionCount();
@@ -154,15 +157,29 @@ $endif$
     }
 
 $if(BytesRef)$
-    private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) {
-        var inputOrds = inputBlock.getOrdinalsBlock();
+    private IntVector addOrdinalsVector(OrdinalBytesRefVector inputBlock) {
+        IntVector inputOrds = inputBlock.getOrdinalsVector();
         try (
-            var builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount());
+            var builder = blockFactory.newIntVectorBuilder(inputOrds.getPositionCount());
             var hashOrds = add(inputBlock.getDictionaryVector())
         ) {
-            for (int i = 0; i < inputOrds.getPositionCount(); i++) {
-                int valueCount = inputOrds.getValueCount(i);
-                int firstIndex = inputOrds.getFirstValueIndex(i);
+            for (int p = 0; p < inputOrds.getPositionCount(); p++) {
+                int ord = hashOrds.getInt(inputOrds.getInt(p));
+                builder.appendInt(ord);
+            }
+            return builder.build();
+        }
+    }
+
+    private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) {
+        try (
+            IntBlock inputOrds = new MultivalueDedupeInt(inputBlock.getOrdinalsBlock()).dedupeToBlockAdaptive(blockFactory);
+            IntBlock.Builder builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount());
+            IntVector hashOrds = add(inputBlock.getDictionaryVector())
+        ) {
+            for (int p = 0; p < inputOrds.getPositionCount(); p++) {
+                int valueCount = inputOrds.getValueCount(p);
+                int firstIndex = inputOrds.getFirstValueIndex(p);
                 switch (valueCount) {
                     case 0 -> {
                         builder.appendInt(0);
@@ -173,9 +190,11 @@ $if(BytesRef)$
                         builder.appendInt(ord);
                     }
                     default -> {
+                        int start = firstIndex;
+                        int end = firstIndex + valueCount;
                         builder.beginPositionEntry();
-                        for (int v = 0; v < valueCount; v++) {
-                            int ord = hashOrds.getInt(inputOrds.getInt(firstIndex + i));
+                        for (int i = start; i < end; i++) {
+                            int ord = hashOrds.getInt(inputOrds.getInt(i));
                             builder.appendInt(ord);
                         }
                         builder.endPositionEntry();
@@ -224,26 +243,21 @@ $if(BytesRef)$
          * without and still read from the block.
          */
         // TODO replace with takeBytesRefsOwnership ?!
+        final BytesRef spare = new BytesRef();
         if (seenNull) {
             try (var builder = blockFactory.newBytesRefBlockBuilder(Math.toIntExact(hash.size() + 1))) {
                 builder.appendNull();
-                BytesRef spare = new BytesRef();
                 for (long i = 0; i < hash.size(); i++) {
                     builder.appendBytesRef(hash.get(i, spare));
                 }
                 return new BytesRefBlock[] { builder.build() };
             }
         }
-
-        final int size = Math.toIntExact(hash.size());
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            hash.getBytesRefs().writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                return new BytesRefBlock[] {
-                    blockFactory.newBytesRefArrayVector(new BytesRefArray(in, BigArrays.NON_RECYCLING_INSTANCE), size).asBlock() };
+        try (var builder = blockFactory.newBytesRefBlockBuilder(Math.toIntExact(hash.size()))) {
+            for (long i = 0; i < hash.size(); i++) {
+                builder.appendBytesRef(hash.get(i, spare));
             }
-        } catch (IOException e) {
-            throw new IllegalStateException(e);
+            return new BytesRefBlock[] { builder.build() };
         }
 $else$
         if (seenNull) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java
index 1fc2430393c98..c66c960dd8a99 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java
@@ -58,7 +58,7 @@ public static void evaluateIntermediate(CentroidState state, DriverContext drive
     }
 
     public static Block evaluateFinal(CentroidState state, DriverContext driverContext) {
-        return driverContext.blockFactory().newConstantBytesRefBlockWith(state.encodeCentroidResult(), 1);
+        return state.toBlock(driverContext.blockFactory());
     }
 
     public static void combineStates(GroupingCentroidState current, int groupId, GroupingCentroidState state, int statePosition) {
@@ -181,10 +181,14 @@ public void add(double x, double dx, double y, double dy, long count) {
             this.count += count;
         }
 
-        protected BytesRef encodeCentroidResult() {
-            double x = xSum.value() / count;
-            double y = ySum.value() / count;
-            return encode(x, y);
+        protected Block toBlock(BlockFactory blockFactory) {
+            if (count > 0) {
+                double x = xSum.value() / count;
+                double y = ySum.value() / count;
+                return blockFactory.newConstantBytesRefBlockWith(encode(x, y), 1);
+            } else {
+                return blockFactory.newConstantNullBlock(1);
+            }
         }
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractArrayBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractArrayBlock.java
index c14289ff2366b..ab125bb868388 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractArrayBlock.java
@@ -191,8 +191,7 @@ void writeSubFields(StreamOutput out) throws IOException {
         if (nullsMask != null) {
             out.writeLongArray(nullsMask.toLongArray());
         }
-        if (out.getTransportVersion().before(TransportVersions.ESQL_MV_ORDERING_SORTED_ASCENDING)
-            && mvOrdering == MvOrdering.SORTED_ASCENDING) {
+        if (out.getTransportVersion().before(TransportVersions.V_8_15_0) && mvOrdering == MvOrdering.SORTED_ASCENDING) {
             out.writeEnum(MvOrdering.UNORDERED);
         } else {
             out.writeEnum(mvOrdering);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java
index 1c9c97a364fc7..9633051781f4a 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java
@@ -120,7 +120,7 @@ static List<List<PartialLeafReaderContext>> docSlices(IndexReader indexReader, i
     }
 
     static List<List<PartialLeafReaderContext>> segmentSlices(List<LeafReaderContext> leafContexts) {
-        IndexSearcher.LeafSlice[] gs = IndexSearcher.slices(leafContexts, MAX_DOCS_PER_SLICE, MAX_SEGMENTS_PER_SLICE);
-        return Arrays.stream(gs).map(g -> Arrays.stream(g.leaves).map(PartialLeafReaderContext::new).toList()).toList();
+        IndexSearcher.LeafSlice[] gs = IndexSearcher.slices(leafContexts, MAX_DOCS_PER_SLICE, MAX_SEGMENTS_PER_SLICE, false);
+        return Arrays.stream(gs).map(g -> Arrays.stream(g.partitions).map(PartialLeafReaderContext::new).toList()).toList();
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java
index 2e32d20a2365e..0f600958b93b3 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.TopFieldCollector;
+import org.apache.lucene.search.TopFieldCollectorManager;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.DocVector;
@@ -230,8 +231,9 @@ static final class PerShardCollector {
             if (sortAndFormats.isEmpty()) {
                 throw new IllegalStateException("sorts must not be disabled in TopN");
             }
+
             // We don't use CollectorManager here as we don't retrieve the total hits and sort by score.
-            this.topFieldCollector = TopFieldCollector.create(sortAndFormats.get().sort, limit, 0);
+            this.topFieldCollector = new TopFieldCollectorManager(sortAndFormats.get().sort, limit, null, 0, false).newCollector();
         }
 
         LeafCollector getLeafCollector(LeafReaderContext leafReaderContext) throws IOException {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/PartialLeafReaderContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/PartialLeafReaderContext.java
index e9063c9597c5f..c92dc75397729 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/PartialLeafReaderContext.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/PartialLeafReaderContext.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.compute.lucene;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
 
 /**
  * A subset of a {@link LeafReaderContext}.
@@ -16,6 +17,10 @@
  * @param maxDoc one more than the last document
  */
 public record PartialLeafReaderContext(LeafReaderContext leafReaderContext, int minDoc, int maxDoc) {
+    public PartialLeafReaderContext(IndexSearcher.LeafReaderContextPartition partition) {
+        this(partition.ctx, partition.minDocId, partition.maxDocId);
+    }
+
     public PartialLeafReaderContext(LeafReaderContext leafReaderContext) {
         this(leafReaderContext, 0, leafReaderContext.reader().maxDoc());
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java
index ee747d98c26f8..74affb10eaf20 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java
@@ -241,6 +241,12 @@ private void loadFromSingleLeaf(Block[] blocks, int shard, int segment, BlockLoa
                 }
             }
 
+            SourceLoader sourceLoader = null;
+            if (storedFieldsSpec.requiresSource()) {
+                sourceLoader = shardContexts.get(shard).newSourceLoader.get();
+                storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields()));
+            }
+
             if (rowStrideReaders.isEmpty()) {
                 return;
             }
@@ -259,7 +265,7 @@ private void loadFromSingleLeaf(Block[] blocks, int shard, int segment, BlockLoa
             }
             BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader(
                 storedFieldLoader.getLoader(ctx, null),
-                storedFieldsSpec.requiresSource() ? shardContexts.get(shard).newSourceLoader.get().leaf(ctx.reader(), null) : null
+                sourceLoader != null ? sourceLoader.leaf(ctx.reader(), null) : null
             );
             for (int p = 0; p < docs.count(); p++) {
                 int doc = docs.get(p);
@@ -381,13 +387,18 @@ private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException {
                 FieldWork field = fields[f];
                 rowStride[f] = field.rowStride(ctx);
                 storedFieldsSpec = storedFieldsSpec.merge(field.loader.rowStrideStoredFieldSpec());
-                storedFields = new BlockLoaderStoredFieldsFromLeafLoader(
-                    StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx, null),
-                    storedFieldsSpec.requiresSource() ? shardContexts.get(shard).newSourceLoader.get().leaf(ctx.reader(), null) : null
-                );
-                if (false == storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) {
-                    trackStoredFields(storedFieldsSpec, false);
-                }
+            }
+            SourceLoader sourceLoader = null;
+            if (storedFieldsSpec.requiresSource()) {
+                sourceLoader = shardContexts.get(shard).newSourceLoader.get();
+                storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields()));
+            }
+            storedFields = new BlockLoaderStoredFieldsFromLeafLoader(
+                StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx, null),
+                sourceLoader != null ? sourceLoader.leaf(ctx.reader(), null) : null
+            );
+            if (false == storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) {
+                trackStoredFields(storedFieldsSpec, false);
             }
         }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java
index 4fb4053b0c0f4..32492af157fe6 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java
@@ -231,7 +231,7 @@ public String toString() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.ESQL_PAGE_MAPPING_TO_ITERATOR;
+            return TransportVersions.V_8_15_0;
         }
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
index e7b16072f4b66..d98613f1817ab 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
@@ -9,11 +9,10 @@
 
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContent;
@@ -168,24 +167,24 @@ public DriverSleeps sleeps() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return Iterators.concat(ChunkedToXContentHelper.startObject(), Iterators.single((b, p) -> {
-            b.timeField("start_millis", "start", startMillis);
-            b.timeField("stop_millis", "stop", stopMillis);
-            b.field("took_nanos", tookNanos);
-            if (b.humanReadable()) {
-                b.field("took_time", TimeValue.timeValueNanos(tookNanos));
-            }
-            b.field("cpu_nanos", cpuNanos);
-            if (b.humanReadable()) {
-                b.field("cpu_time", TimeValue.timeValueNanos(cpuNanos));
-            }
-            b.field("iterations", iterations);
-            return b;
-        }),
-            ChunkedToXContentHelper.array("operators", operators.iterator()),
-            Iterators.single((b, p) -> b.field("sleeps", sleeps)),
-            ChunkedToXContentHelper.endObject()
-        );
+        return ChunkedToXContent.builder(params).object(ob -> {
+            ob.append((b, p) -> {
+                b.timestampFieldsFromUnixEpochMillis("start_millis", "start", startMillis);
+                b.timestampFieldsFromUnixEpochMillis("stop_millis", "stop", stopMillis);
+                b.field("took_nanos", tookNanos);
+                if (b.humanReadable()) {
+                    b.field("took_time", TimeValue.timeValueNanos(tookNanos));
+                }
+                b.field("cpu_nanos", cpuNanos);
+                if (b.humanReadable()) {
+                    b.field("cpu_time", TimeValue.timeValueNanos(cpuNanos));
+                }
+                b.field("iterations", iterations);
+                return b;
+            });
+            ob.array("operators", operators.iterator());
+            ob.field("sleeps", sleeps);
+        });
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java
index 217a0b033bed4..01e9a73c4fb5f 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java
@@ -62,9 +62,9 @@ public boolean isStillSleeping() {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("reason", reason);
-            builder.timeField("sleep_millis", "sleep", sleep);
+            builder.timestampFieldsFromUnixEpochMillis("sleep_millis", "sleep", sleep);
             if (wake > 0) {
-                builder.timeField("wake_millis", "wake", wake);
+                builder.timestampFieldsFromUnixEpochMillis("wake_millis", "wake", wake);
             }
             return builder.endObject();
         }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java
index 5e0e625abb914..7cf47bc7fed1c 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java
@@ -372,8 +372,8 @@ static final class OrdinalSegmentAggregator implements Releasable, SeenGroupIds
         }
 
         void addInput(IntVector docs, Page page) {
+            GroupingAggregatorFunction.AddInput[] prepared = new GroupingAggregatorFunction.AddInput[aggregators.size()];
             try {
-                GroupingAggregatorFunction.AddInput[] prepared = new GroupingAggregatorFunction.AddInput[aggregators.size()];
                 for (int i = 0; i < prepared.length; i++) {
                     prepared[i] = aggregators.get(i).prepareProcessPage(this, page);
                 }
@@ -392,7 +392,7 @@ void addInput(IntVector docs, Page page) {
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
             } finally {
-                page.releaseBlocks();
+                Releasables.close(page::releaseBlocks, Releasables.wrap(prepared));
             }
         }
 
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java
index a4eb252dbf35c..e2c9c255b67bd 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.data.TestBlockFactory;
+import org.elasticsearch.compute.operator.AddGarbageRowsSourceOperator;
 import org.elasticsearch.compute.operator.AggregationOperator;
 import org.elasticsearch.compute.operator.CannedSourceOperator;
 import org.elasticsearch.compute.operator.Driver;
@@ -203,6 +204,22 @@ public void testNoneFiltered() {
         assertSimpleOutput(origInput, results);
     }
 
+    public void testSomeFiltered() {
+        Operator.OperatorFactory factory = simpleWithMode(
+            AggregatorMode.SINGLE,
+            agg -> new FilteredAggregatorFunctionSupplier(agg, AddGarbageRowsSourceOperator.filterFactory())
+        );
+        DriverContext driverContext = driverContext();
+        // Build the test data
+        List<Page> input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10));
+        List<Page> origInput = BlockTestUtils.deepCopyOf(input, TestBlockFactory.getNonBreakingInstance());
+        // Sprinkle garbage into it
+        input = CannedSourceOperator.collectPages(new AddGarbageRowsSourceOperator(new CannedSourceOperator(input.iterator())));
+        List<Page> results = drive(factory.get(driverContext), input.iterator(), driverContext);
+        assertThat(results, hasSize(1));
+        assertSimpleOutput(origInput, results);
+    }
+
     // Returns an intermediate state that is equivalent to what the local execution planner will emit
     // if it determines that certain shards have no relevant data.
     List<Page> nullIntermediateState(BlockFactory blockFactory) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java
index 7e1575fb81726..35ecced470e01 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java
@@ -27,7 +27,6 @@
 public class FilteredAggregatorFunctionTests extends AggregatorFunctionTestCase {
     private final List<Exception> unclosed = Collections.synchronizedList(new ArrayList<>());
 
-    // TODO some version of this test that applies across all aggs
     @Override
     protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
         return new FilteredAggregatorFunctionSupplier(
@@ -103,4 +102,9 @@ public void testNoneFiltered() {
     public void testAllFiltered() {
         assumeFalse("can't double filter. tests already filter.", true);
     }
+
+    @Override
+    public void testSomeFiltered() {
+        assumeFalse("can't double filter. tests already filter.", true);
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunctionTests.java
index 87cb99bd0709f..26971dc927cd1 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunctionTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredGroupingAggregatorFunctionTests.java
@@ -11,12 +11,14 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanVector;
 import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.LongIntBlockSourceOperator;
 import org.elasticsearch.compute.operator.SourceOperator;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.Tuple;
 import org.junit.After;
 
@@ -31,7 +33,6 @@
 public class FilteredGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase {
     private final List<Exception> unclosed = Collections.synchronizedList(new ArrayList<>());
 
-    // TODO some version of this test that applies across all aggs
     @Override
     protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
         return new FilteredAggregatorFunctionSupplier(
@@ -104,6 +105,42 @@ protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
         );
     }
 
+    /**
+     * Tests {@link GroupingAggregator#addIntermediateRow} by building results using the traditional
+     * add mechanism and using {@link GroupingAggregator#addIntermediateRow} then asserting that they
+     * produce the same output.
+     */
+    public void testAddIntermediateRowInput() {
+        DriverContext ctx = driverContext();
+        AggregatorFunctionSupplier supplier = aggregatorFunction(channels(AggregatorMode.SINGLE));
+        Block[] results = new Block[2];
+        try (
+            GroupingAggregatorFunction main = supplier.groupingAggregator(ctx);
+            GroupingAggregatorFunction leaf = supplier.groupingAggregator(ctx);
+            SourceOperator source = simpleInput(ctx.blockFactory(), 10);
+        ) {
+            Page p;
+            while ((p = source.getOutput()) != null) {
+                try (
+                    IntVector group = ctx.blockFactory().newConstantIntVector(0, p.getPositionCount());
+                    GroupingAggregatorFunction.AddInput addInput = leaf.prepareProcessPage(null, p)
+                ) {
+                    addInput.add(0, group);
+                } finally {
+                    p.releaseBlocks();
+                }
+            }
+            main.addIntermediateRowInput(0, leaf, 0);
+            try (IntVector selected = ctx.blockFactory().newConstantIntVector(0, 1)) {
+                main.evaluateFinal(results, 0, selected, ctx);
+                leaf.evaluateFinal(results, 1, selected, ctx);
+            }
+            assertThat(results[0], equalTo(results[1]));
+        } finally {
+            Releasables.close(results);
+        }
+    }
+
     @After
     public void checkUnclosed() {
         for (Exception tracker : unclosed) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java
index 316058e57e089..cb190dfffafb9 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.data.TestBlockFactory;
+import org.elasticsearch.compute.operator.AddGarbageRowsSourceOperator;
 import org.elasticsearch.compute.operator.CannedSourceOperator;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.ForkingOperatorTestCase;
@@ -53,6 +54,7 @@
 import static org.elasticsearch.compute.data.BlockTestUtils.append;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.in;
 
 /**
  * Shared tests for testing grouped aggregations.
@@ -87,14 +89,17 @@ protected final Operator.OperatorFactory simpleWithMode(AggregatorMode mode) {
         return simpleWithMode(mode, Function.identity());
     }
 
+    protected List<Integer> channels(AggregatorMode mode) {
+        return mode.isInputPartial() ? range(1, 1 + aggregatorIntermediateBlockCount()).boxed().toList() : List.of(1);
+    }
+
     private Operator.OperatorFactory simpleWithMode(
         AggregatorMode mode,
         Function<AggregatorFunctionSupplier, AggregatorFunctionSupplier> wrap
     ) {
-        List<Integer> channels = mode.isInputPartial() ? range(1, 1 + aggregatorIntermediateBlockCount()).boxed().toList() : List.of(1);
         int emitChunkSize = between(100, 200);
 
-        AggregatorFunctionSupplier supplier = wrap.apply(aggregatorFunction(channels));
+        AggregatorFunctionSupplier supplier = wrap.apply(aggregatorFunction(channels(mode)));
         if (randomBoolean()) {
             supplier = chunkGroups(emitChunkSize, supplier);
         }
@@ -160,11 +165,17 @@ protected long randomGroupId(int pageSize) {
 
     @Override
     protected final void assertSimpleOutput(List<Page> input, List<Page> results) {
+        assertSimpleOutput(input, results, true);
+    }
+
+    private void assertSimpleOutput(List<Page> input, List<Page> results, boolean assertGroupCount) {
         SeenGroups seenGroups = seenGroups(input);
 
         assertThat(results, hasSize(1));
         assertThat(results.get(0).getBlockCount(), equalTo(2));
-        assertThat(results.get(0).getPositionCount(), equalTo(seenGroups.size()));
+        if (assertGroupCount) {
+            assertThat(results.get(0).getPositionCount(), equalTo(seenGroups.size()));
+        }
 
         Block groups = results.get(0).getBlock(0);
         Block result = results.get(0).getBlock(1);
@@ -394,6 +405,23 @@ public final void testNoneFiltered() {
         assertSimpleOutput(origInput, results);
     }
 
+    public void testSomeFiltered() {
+        Operator.OperatorFactory factory = simpleWithMode(
+            AggregatorMode.SINGLE,
+            agg -> new FilteredAggregatorFunctionSupplier(agg, AddGarbageRowsSourceOperator.filterFactory())
+        );
+        DriverContext driverContext = driverContext();
+        // Build the test data
+        List<Page> input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10));
+        List<Page> origInput = BlockTestUtils.deepCopyOf(input, TestBlockFactory.getNonBreakingInstance());
+        // Sprinkle garbage into it
+        input = CannedSourceOperator.collectPages(new AddGarbageRowsSourceOperator(new CannedSourceOperator(input.iterator())));
+        List<Page> results = drive(factory.get(driverContext), input.iterator(), driverContext);
+        assertThat(results, hasSize(1));
+
+        assertSimpleOutput(origInput, results, false);
+    }
+
     /**
      * Asserts that the output from an empty input is a {@link Block} containing
      * only {@code null}. Override for {@code count} style aggregations that
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java
index 27ec0b979e8ae..42ac4cf2ff917 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java
@@ -21,10 +21,13 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BlockTestUtils;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.MockBlockFactory;
+import org.elasticsearch.compute.data.OrdinalBytesRefBlock;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.data.TestBlockFactory;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeTests;
 import org.elasticsearch.core.ReleasableIterator;
 import org.elasticsearch.core.Releasables;
@@ -38,11 +41,13 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.test.ListMatcher.matchesList;
 import static org.elasticsearch.test.MapMatcher.assertMap;
@@ -58,26 +63,40 @@
 public class BlockHashRandomizedTests extends ESTestCase {
     @ParametersFactory
     public static List<Object[]> params() {
-        List<Object[]> params = new ArrayList<>();
+        List<List<? extends Type>> allowedTypesChoices = List.of(
+            /*
+             * Run with only `LONG` elements because we have some
+             * optimizations that hit if you only have those.
+             */
+            List.of(new Basic(ElementType.LONG)),
+            /*
+             * Run with only `BYTES_REF` elements because we have some
+             * optimizations that hit if you only have those.
+             */
+            List.of(new Basic(ElementType.BYTES_REF)),
+            /*
+             * Run with only `BYTES_REF` elements in an OrdinalBytesRefBlock
+             * because we have a few optimizations that use it.
+             */
+            List.of(new Ordinals(10)),
+            /*
+             * Run with only `LONG` and `BYTES_REF` elements because
+             * we have some optimizations that hit if you only have
+             * those.
+             */
+            List.of(new Basic(ElementType.LONG), new Basic(ElementType.BYTES_REF)),
+            /*
+             * Any random source.
+             */
+            Stream.concat(Stream.of(new Ordinals(10)), MultivalueDedupeTests.supportedTypes().stream().map(Basic::new)).toList()
+        );
 
+        List<Object[]> params = new ArrayList<>();
         for (boolean forcePackedHash : new boolean[] { false, true }) {
             for (int groups : new int[] { 1, 2, 3, 4, 5, 10 }) {
                 for (int maxValuesPerPosition : new int[] { 1, 3 }) {
                     for (int dups : new int[] { 0, 2 }) {
-                        for (List<ElementType> allowedTypes : List.of(
-                            /*
-                             * Run with only `LONG` elements because we have some
-                             * optimizations that hit if you only have those.
-                             */
-                            List.of(ElementType.LONG),
-                            /*
-                             * Run with only `LONG` and `BYTES_REF` elements because
-                             * we have some optimizations that hit if you only have
-                             * those.
-                             */
-                            List.of(ElementType.LONG, ElementType.BYTES_REF),
-                            MultivalueDedupeTests.supportedTypes()
-                        )) {
+                        for (List<? extends Type> allowedTypes : allowedTypesChoices) {
                             params.add(new Object[] { forcePackedHash, groups, maxValuesPerPosition, dups, allowedTypes });
                         }
                     }
@@ -87,18 +106,33 @@ public static List<Object[]> params() {
         return params;
     }
 
+    /**
+     * The type of {@link Block} being tested.
+     */
+    interface Type {
+        /**
+         * The type of the {@link ElementType elements} in the {@link Block}.
+         */
+        ElementType elementType();
+
+        /**
+         * Build a random {@link Block}.
+         */
+        BasicBlockTests.RandomBlock randomBlock(int positionCount, int maxValuesPerPosition, int dups);
+    }
+
     private final boolean forcePackedHash;
     private final int groups;
     private final int maxValuesPerPosition;
     private final int dups;
-    private final List<ElementType> allowedTypes;
+    private final List<? extends Type> allowedTypes;
 
     public BlockHashRandomizedTests(
         @Name("forcePackedHash") boolean forcePackedHash,
         @Name("groups") int groups,
         @Name("maxValuesPerPosition") int maxValuesPerPosition,
         @Name("dups") int dups,
-        @Name("allowedTypes") List<ElementType> allowedTypes
+        @Name("allowedTypes") List<Type> allowedTypes
     ) {
         this.forcePackedHash = forcePackedHash;
         this.groups = groups;
@@ -127,21 +161,22 @@ public void testWithCranky() {
     }
 
     private void test(MockBlockFactory blockFactory) {
-        List<ElementType> types = randomList(groups, groups, () -> randomFrom(allowedTypes));
+        List<Type> types = randomList(groups, groups, () -> randomFrom(allowedTypes));
+        List<ElementType> elementTypes = types.stream().map(Type::elementType).toList();
         BasicBlockTests.RandomBlock[] randomBlocks = new BasicBlockTests.RandomBlock[types.size()];
         Block[] blocks = new Block[types.size()];
-        int pageCount = between(1, 10);
+        int pageCount = between(1, groups < 10 ? 10 : 5);
         int positionCount = 100;
         int emitBatchSize = 100;
-        try (BlockHash blockHash = newBlockHash(blockFactory, emitBatchSize, types)) {
+        try (BlockHash blockHash = newBlockHash(blockFactory, emitBatchSize, elementTypes)) {
             /*
              * Only the long/long, long/bytes_ref, and bytes_ref/long implementations don't collect nulls.
              */
             Oracle oracle = new Oracle(
                 forcePackedHash
-                    || false == (types.equals(List.of(ElementType.LONG, ElementType.LONG))
-                        || types.equals(List.of(ElementType.LONG, ElementType.BYTES_REF))
-                        || types.equals(List.of(ElementType.BYTES_REF, ElementType.LONG)))
+                    || false == (elementTypes.equals(List.of(ElementType.LONG, ElementType.LONG))
+                        || elementTypes.equals(List.of(ElementType.LONG, ElementType.BYTES_REF))
+                        || elementTypes.equals(List.of(ElementType.BYTES_REF, ElementType.LONG)))
             );
             /*
              * Expected ordinals for checking lookup. Skipped if we have more than 5 groups because
@@ -151,15 +186,7 @@ private void test(MockBlockFactory blockFactory) {
 
             for (int p = 0; p < pageCount; p++) {
                 for (int g = 0; g < blocks.length; g++) {
-                    randomBlocks[g] = BasicBlockTests.randomBlock(
-                        types.get(g),
-                        positionCount,
-                        types.get(g) == ElementType.NULL ? true : randomBoolean(),
-                        1,
-                        maxValuesPerPosition,
-                        0,
-                        dups
-                    );
+                    randomBlocks[g] = types.get(g).randomBlock(positionCount, maxValuesPerPosition, dups);
                     blocks[g] = randomBlocks[g].block();
                 }
                 oracle.add(randomBlocks);
@@ -209,6 +236,7 @@ private void test(MockBlockFactory blockFactory) {
 
                 if (blockHash instanceof LongLongBlockHash == false
                     && blockHash instanceof BytesRefLongBlockHash == false
+                    && blockHash instanceof BytesRef2BlockHash == false
                     && blockHash instanceof BytesRef3BlockHash == false) {
                     assertLookup(blockFactory, expectedOrds, types, blockHash, oracle);
                 }
@@ -235,14 +263,14 @@ private BlockHash newBlockHash(BlockFactory blockFactory, int emitBatchSize, Lis
     private void assertLookup(
         BlockFactory blockFactory,
         Map<List<Object>, Set<Integer>> expectedOrds,
-        List<ElementType> types,
+        List<Type> types,
         BlockHash blockHash,
         Oracle oracle
     ) {
         Block.Builder[] builders = new Block.Builder[types.size()];
         try {
             for (int b = 0; b < builders.length; b++) {
-                builders[b] = types.get(b).newBlockBuilder(LOOKUP_POSITIONS, blockFactory);
+                builders[b] = types.get(b).elementType().newBlockBuilder(LOOKUP_POSITIONS, blockFactory);
             }
             for (int p = 0; p < LOOKUP_POSITIONS; p++) {
                 /*
@@ -408,8 +436,8 @@ static CircuitBreakerService mockBreakerService(CircuitBreaker breaker) {
         return breakerService;
     }
 
-    private static List<Object> randomKey(List<ElementType> types) {
-        return types.stream().map(BlockHashRandomizedTests::randomKeyElement).toList();
+    private static List<Object> randomKey(List<Type> types) {
+        return types.stream().map(t -> randomKeyElement(t.elementType())).toList();
     }
 
     public static Object randomKeyElement(ElementType type) {
@@ -423,4 +451,76 @@ public static Object randomKeyElement(ElementType type) {
             default -> throw new IllegalArgumentException("unsupported element type [" + type + "]");
         };
     }
+
+    private record Basic(ElementType elementType) implements Type {
+        @Override
+        public BasicBlockTests.RandomBlock randomBlock(int positionCount, int maxValuesPerPosition, int dups) {
+            return BasicBlockTests.randomBlock(
+                elementType,
+                positionCount,
+                elementType == ElementType.NULL | randomBoolean(),
+                1,
+                maxValuesPerPosition,
+                0,
+                dups
+            );
+        }
+    }
+
+    private record Ordinals(int dictionarySize) implements Type {
+        @Override
+        public ElementType elementType() {
+            return ElementType.BYTES_REF;
+        }
+
+        @Override
+        public BasicBlockTests.RandomBlock randomBlock(int positionCount, int maxValuesPerPosition, int dups) {
+            Map<String, Integer> dictionary = new HashMap<>();
+            Set<String> keys = dictionary(maxValuesPerPosition);
+            List<List<Object>> values = new ArrayList<>(positionCount);
+            try (
+                IntBlock.Builder ordinals = TestBlockFactory.getNonBreakingInstance()
+                    .newIntBlockBuilder(positionCount * maxValuesPerPosition);
+                BytesRefVector.Builder bytes = TestBlockFactory.getNonBreakingInstance().newBytesRefVectorBuilder(maxValuesPerPosition);
+            ) {
+                for (int p = 0; p < positionCount; p++) {
+                    int valueCount = between(1, maxValuesPerPosition);
+                    int dupCount = between(0, dups);
+
+                    List<Integer> ordsAtPosition = new ArrayList<>();
+                    List<Object> valuesAtPosition = new ArrayList<>();
+                    values.add(valuesAtPosition);
+                    if (valueCount != 1 || dupCount != 0) {
+                        ordinals.beginPositionEntry();
+                    }
+                    for (int v = 0; v < valueCount; v++) {
+                        String key = randomFrom(keys);
+                        int ordinal = dictionary.computeIfAbsent(key, k -> {
+                            bytes.appendBytesRef(new BytesRef(k));
+                            return dictionary.size();
+                        });
+                        valuesAtPosition.add(new BytesRef(key));
+                        ordinals.appendInt(ordinal);
+                        ordsAtPosition.add(ordinal);
+                    }
+                    for (int v = 0; v < dupCount; v++) {
+                        ordinals.appendInt(randomFrom(ordsAtPosition));
+                    }
+                    if (valueCount != 1 || dupCount != 0) {
+                        ordinals.endPositionEntry();
+                    }
+                }
+                return new BasicBlockTests.RandomBlock(values, new OrdinalBytesRefBlock(ordinals.build(), bytes.build()));
+            }
+        }
+
+        private Set<String> dictionary(int maxValuesPerPosition) {
+            int count = Math.max(dictionarySize, maxValuesPerPosition);
+            Set<String> values = new HashSet<>();
+            while (values.size() < count) {
+                values.add(randomAlphaOfLength(5));
+            }
+            return values;
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java
index 800683c696c0f..088e791348840 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java
@@ -20,12 +20,15 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.MockBlockFactory;
+import org.elasticsearch.compute.data.OrdinalBytesRefBlock;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.data.TestBlockFactory;
 import org.elasticsearch.core.Releasable;
@@ -460,6 +463,133 @@ public void testBytesRefHashWithMultiValuedFields() {
         }
     }
 
+    public void testBasicOrdinals() {
+        try (
+            IntVector.Builder ords = blockFactory.newIntVectorFixedBuilder(8);
+            BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(8)
+        ) {
+            ords.appendInt(1);
+            ords.appendInt(0);
+            ords.appendInt(3);
+            ords.appendInt(1);
+            ords.appendInt(3);
+            ords.appendInt(0);
+            ords.appendInt(2);
+            ords.appendInt(3);
+            bytes.appendBytesRef(new BytesRef("item-1"));
+            bytes.appendBytesRef(new BytesRef("item-2"));
+            bytes.appendBytesRef(new BytesRef("item-3"));
+            bytes.appendBytesRef(new BytesRef("item-4"));
+
+            hash(ordsAndKeys -> {
+                if (forcePackedHash) {
+                    assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size="));
+                    assertThat(ordsAndKeys.description, endsWith("b}"));
+                    assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2);
+                    assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4)));
+                    assertKeys(ordsAndKeys.keys, "item-2", "item-1", "item-4", "item-3");
+                } else {
+                    assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=4, size="));
+                    assertThat(ordsAndKeys.description, endsWith("b, seenNull=false}"));
+                    assertOrds(ordsAndKeys.ords, 2, 1, 4, 2, 4, 1, 3, 4);
+                    assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5)));
+                    assertKeys(ordsAndKeys.keys, "item-1", "item-2", "item-3", "item-4");
+                }
+            }, new OrdinalBytesRefVector(ords.build(), bytes.build()).asBlock());
+        }
+    }
+
+    public void testOrdinalsWithNulls() {
+        try (
+            IntBlock.Builder ords = blockFactory.newIntBlockBuilder(4);
+            BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(2)
+        ) {
+            ords.appendInt(0);
+            ords.appendNull();
+            ords.appendInt(1);
+            ords.appendNull();
+            bytes.appendBytesRef(new BytesRef("cat"));
+            bytes.appendBytesRef(new BytesRef("dog"));
+
+            hash(ordsAndKeys -> {
+                if (forcePackedHash) {
+                    assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=3, size="));
+                    assertThat(ordsAndKeys.description, endsWith("b}"));
+                    assertOrds(ordsAndKeys.ords, 0, 1, 2, 1);
+                    assertKeys(ordsAndKeys.keys, "cat", null, "dog");
+                } else {
+                    assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=2, size="));
+                    assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}"));
+                    assertOrds(ordsAndKeys.ords, 1, 0, 2, 0);
+                    assertKeys(ordsAndKeys.keys, null, "cat", "dog");
+                }
+                assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3)));
+            }, new OrdinalBytesRefBlock(ords.build(), bytes.build()));
+        }
+    }
+
+    public void testOrdinalsWithMultiValuedFields() {
+        try (
+            IntBlock.Builder ords = blockFactory.newIntBlockBuilder(4);
+            BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(2)
+        ) {
+            ords.appendInt(0);
+            ords.beginPositionEntry();
+            ords.appendInt(0);
+            ords.appendInt(1);
+            ords.endPositionEntry();
+            ords.beginPositionEntry();
+            ords.appendInt(1);
+            ords.appendInt(2);
+            ords.endPositionEntry();
+            ords.beginPositionEntry();
+            ords.appendInt(2);
+            ords.appendInt(1);
+            ords.endPositionEntry();
+            ords.appendNull();
+            ords.beginPositionEntry();
+            ords.appendInt(2);
+            ords.appendInt(2);
+            ords.appendInt(1);
+            ords.endPositionEntry();
+
+            bytes.appendBytesRef(new BytesRef("foo"));
+            bytes.appendBytesRef(new BytesRef("bar"));
+            bytes.appendBytesRef(new BytesRef("bort"));
+
+            hash(ordsAndKeys -> {
+                if (forcePackedHash) {
+                    assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size="));
+                    assertThat(ordsAndKeys.description, endsWith("b}"));
+                    assertOrds(
+                        ordsAndKeys.ords,
+                        new int[] { 0 },
+                        new int[] { 0, 1 },
+                        new int[] { 1, 2 },
+                        new int[] { 2, 1 },
+                        new int[] { 3 },
+                        new int[] { 2, 1 }
+                    );
+                    assertKeys(ordsAndKeys.keys, "foo", "bar", "bort", null);
+                } else {
+                    assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=3, size="));
+                    assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}"));
+                    assertOrds(
+                        ordsAndKeys.ords,
+                        new int[] { 1 },
+                        new int[] { 1, 2 },
+                        new int[] { 2, 3 },
+                        new int[] { 3, 2 },
+                        new int[] { 0 },
+                        new int[] { 3, 2 }
+                    );
+                    assertKeys(ordsAndKeys.keys, null, "foo", "bar", "bort");
+                }
+                assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4)));
+            }, new OrdinalBytesRefBlock(ords.build(), bytes.build()));
+        }
+    }
+
     public void testBooleanHashFalseFirst() {
         boolean[] values = new boolean[] { false, true, true, true, true };
         hash(ordsAndKeys -> {
@@ -1017,7 +1147,7 @@ public void testLongBytesRefHashWithMultiValuedFields() {
                 } else {
                     assertThat(
                         ordsAndKeys.description,
-                        equalTo("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=9, size=491b}")
+                        equalTo("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=9, size=483b}")
                     );
                     assertOrds(
                         ordsAndKeys.ords,
@@ -1315,6 +1445,7 @@ public void close() {
         });
         if (blockHash instanceof LongLongBlockHash == false
             && blockHash instanceof BytesRefLongBlockHash == false
+            && blockHash instanceof BytesRef2BlockHash == false
             && blockHash instanceof BytesRef3BlockHash == false) {
             Block[] keys = blockHash.getKeys();
             try (ReleasableIterator<IntBlock> lookup = blockHash.lookup(new Page(keys), ByteSizeValue.ofKb(between(1, 100)))) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
index ccc3dea78adc8..f6d81af7c14e5 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
@@ -26,6 +26,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
@@ -546,7 +547,10 @@ public String indexName() {
 
             @Override
             public IndexSettings indexSettings() {
-                throw new UnsupportedOperationException();
+                var imd = IndexMetadata.builder("test_index")
+                    .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY))
+                    .build();
+                return new IndexSettings(imd, Settings.EMPTY);
             }
 
             @Override
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java
index 848415c4490fa..c8dd6f87be5fc 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java
@@ -24,9 +24,11 @@
 import org.apache.lucene.tests.mockfile.HandleLimitFS;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -500,7 +502,10 @@ public String indexName() {
 
             @Override
             public IndexSettings indexSettings() {
-                throw new UnsupportedOperationException();
+                var imd = IndexMetadata.builder("test_index")
+                    .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY))
+                    .build();
+                return new IndexSettings(imd, Settings.EMPTY);
             }
 
             @Override
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AddGarbageRowsSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AddGarbageRowsSourceOperator.java
new file mode 100644
index 0000000000000..079be87161421
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AddGarbageRowsSourceOperator.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.FloatBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.test.ESTestCase;
+
+/**
+ * A {@link SourceOperator} that inserts random garbage into data from another
+ * {@link SourceOperator}. It also inserts an extra channel at the end of the page
+ * containing a {@code boolean} column. If it is {@code true} then the data came
+ * from the original operator. If it's {@code false} then the data is random
+ * garbage inserted by this operator.
+ */
+public class AddGarbageRowsSourceOperator extends SourceOperator {
+    public static EvalOperator.ExpressionEvaluator.Factory filterFactory() {
+        /*
+         * Grabs the filter from the last block. That's where we put it.
+         */
+        return ctx -> new EvalOperator.ExpressionEvaluator() {
+            @Override
+            public Block eval(Page page) {
+                Block block = page.getBlock(page.getBlockCount() - 1);
+                block.incRef();
+                return block;
+            }
+
+            @Override
+            public void close() {}
+        };
+    }
+
+    private final SourceOperator next;
+
+    public AddGarbageRowsSourceOperator(SourceOperator next) {
+        this.next = next;
+    }
+
+    @Override
+    public void finish() {
+        next.finish();
+    }
+
+    @Override
+    public boolean isFinished() {
+        return next.isFinished();
+    }
+
+    @Override
+    public Page getOutput() {
+        Page page = next.getOutput();
+        if (page == null) {
+            return null;
+        }
+        Block.Builder[] newBlocks = new Block.Builder[page.getBlockCount() + 1];
+        try {
+            for (int b = 0; b < page.getBlockCount(); b++) {
+                Block block = page.getBlock(b);
+                newBlocks[b] = block.elementType().newBlockBuilder(page.getPositionCount(), block.blockFactory());
+            }
+            newBlocks[page.getBlockCount()] = page.getBlock(0).blockFactory().newBooleanBlockBuilder(page.getPositionCount());
+
+            for (int p = 0; p < page.getPositionCount(); p++) {
+                if (ESTestCase.randomBoolean()) {
+                    insertGarbageRows(newBlocks, page);
+                }
+                copyPosition(newBlocks, page, p);
+                if (ESTestCase.randomBoolean()) {
+                    insertGarbageRows(newBlocks, page);
+                }
+            }
+
+            return new Page(Block.Builder.buildAll(newBlocks));
+        } finally {
+            Releasables.close(Releasables.wrap(newBlocks), page::releaseBlocks);
+        }
+    }
+
+    private void copyPosition(Block.Builder[] newBlocks, Page page, int p) {
+        for (int b = 0; b < page.getBlockCount(); b++) {
+            Block block = page.getBlock(b);
+            newBlocks[b].copyFrom(block, p, p + 1);
+        }
+        signalKeep(newBlocks, true);
+    }
+
+    private void insertGarbageRows(Block.Builder[] newBlocks, Page page) {
+        int count = ESTestCase.between(1, 5);
+        for (int c = 0; c < count; c++) {
+            insertGarbageRow(newBlocks, page);
+        }
+    }
+
+    private void insertGarbageRow(Block.Builder[] newBlocks, Page page) {
+        for (int b = 0; b < page.getBlockCount(); b++) {
+            Block block = page.getBlock(b);
+            switch (block.elementType()) {
+                case BOOLEAN -> ((BooleanBlock.Builder) newBlocks[b]).appendBoolean(ESTestCase.randomBoolean());
+                case BYTES_REF -> ((BytesRefBlock.Builder) newBlocks[b]).appendBytesRef(new BytesRef(ESTestCase.randomAlphaOfLength(5)));
+                case COMPOSITE, DOC, UNKNOWN -> throw new UnsupportedOperationException();
+                case INT -> ((IntBlock.Builder) newBlocks[b]).appendInt(ESTestCase.randomInt());
+                case LONG -> ((LongBlock.Builder) newBlocks[b]).appendLong(ESTestCase.randomLong());
+                case NULL -> newBlocks[b].appendNull();
+                case DOUBLE -> ((DoubleBlock.Builder) newBlocks[b]).appendDouble(ESTestCase.randomDouble());
+                case FLOAT -> ((FloatBlock.Builder) newBlocks[b]).appendFloat(ESTestCase.randomFloat());
+            }
+        }
+        signalKeep(newBlocks, false);
+    }
+
+    private void signalKeep(Block.Builder[] newBlocks, boolean shouldKeep) {
+        ((BooleanBlock.Builder) newBlocks[newBlocks.length - 1]).appendBoolean(shouldKeep);
+    }
+
+    @Override
+    public void close() {
+        next.close();
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
index ab785e739d080..9e07f9c8f5faf 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
@@ -441,7 +441,7 @@ public void sendResponse(TransportResponse transportResponse) {
             PlainActionFuture<Void> sourceCompletionFuture = new PlainActionFuture<>();
             sourceHandler.addCompletionListener(sourceCompletionFuture);
             ExchangeSinkHandler sinkHandler = exchange1.createSinkHandler(exchangeId, randomIntBetween(1, 128));
-            Transport.Connection connection = node0.getConnection(node1.getLocalDiscoNode());
+            Transport.Connection connection = node0.getConnection(node1.getLocalNode());
             sourceHandler.addRemoteSink(exchange0.newRemoteSink(task, exchangeId, node0, connection), randomIntBetween(1, 5));
             Exception err = expectThrows(
                 Exception.class,
diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
index e4223f03c3a03..fb47255e8d52e 100644
--- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
+++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
@@ -1,5 +1,6 @@
 
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
@@ -27,7 +28,7 @@ GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest")
 
 // ESQL is available in 8.11 or later
 def supportedVersion = bwcVersion -> {
-  return bwcVersion.onOrAfter(Version.fromString("8.11.0"));
+  return bwcVersion.onOrAfter(Version.fromString("8.11.0")) && bwcVersion != VersionProperties.elasticsearchVersion
 }
 
 BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName ->
diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
index d0d6d5fa49c42..0e23b29172c32 100644
--- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
@@ -86,4 +86,9 @@ protected boolean supportsAsync() {
     protected boolean enableRoundingDoubleValuesOnAsserting() {
         return true;
     }
+
+    @Override
+    protected boolean supportsInferenceTestService() {
+        return false;
+    }
 }
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
index 3e799730f7269..8446ac63f43a1 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
@@ -261,4 +261,9 @@ static boolean hasIndexMetadata(String query) {
     protected boolean enableRoundingDoubleValuesOnAsserting() {
         return true;
     }
+
+    @Override
+    protected boolean supportsInferenceTestService() {
+        return false;
+    }
 }
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
index 454f3962c07ea..1f72827057c5b 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
@@ -127,8 +127,18 @@ void indexDocs(RestClient client, String index, List<Doc> docs) throws IOExcepti
         refresh(client, index);
     }
 
-    private Map<String, Object> run(String query) throws IOException {
-        Map<String, Object> resp = runEsql(new RestEsqlTestCase.RequestObjectBuilder().query(query).build());
+    private Map<String, Object> run(String query, boolean includeCCSMetadata) throws IOException {
+        Map<String, Object> resp = runEsql(
+            new RestEsqlTestCase.RequestObjectBuilder().query(query).includeCCSMetadata(includeCCSMetadata).build()
+        );
+        logger.info("--> query {} response {}", query, resp);
+        return resp;
+    }
+
+    private Map<String, Object> runWithColumnarAndIncludeCCSMetadata(String query) throws IOException {
+        Map<String, Object> resp = runEsql(
+            new RestEsqlTestCase.RequestObjectBuilder().query(query).includeCCSMetadata(true).columnar(true).build()
+        );
         logger.info("--> query {} response {}", query, resp);
         return resp;
     }
@@ -147,62 +157,77 @@ private Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder reques
 
     public void testCount() throws Exception {
         {
-            Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS c = COUNT(*)");
+            boolean includeCCSMetadata = randomBoolean();
+            Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS c = COUNT(*)", includeCCSMetadata);
             var columns = List.of(Map.of("name", "c", "type", "long"));
             var values = List.of(List.of(localDocs.size() + remoteDocs.size()));
 
             MapMatcher mapMatcher = matchesMap();
-            assertMap(
-                result,
-                mapMatcher.entry("columns", columns)
-                    .entry("values", values)
-                    .entry("took", greaterThanOrEqualTo(0))
-                    .entry("_clusters", any(Map.class))
-            );
-            assertClusterDetailsMap(result, false);
+            if (includeCCSMetadata) {
+                mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
+            }
+            assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
+            if (includeCCSMetadata) {
+                assertClusterDetailsMap(result, false);
+            }
         }
         {
-            Map<String, Object> result = run("FROM *:test-remote-index | STATS c = COUNT(*)");
+            boolean includeCCSMetadata = randomBoolean();
+            Map<String, Object> result = run("FROM *:test-remote-index | STATS c = COUNT(*)", includeCCSMetadata);
             var columns = List.of(Map.of("name", "c", "type", "long"));
             var values = List.of(List.of(remoteDocs.size()));
 
             MapMatcher mapMatcher = matchesMap();
-            assertMap(
-                result,
-                mapMatcher.entry("columns", columns)
-                    .entry("values", values)
-                    .entry("took", greaterThanOrEqualTo(0))
-                    .entry("_clusters", any(Map.class))
-            );
-            assertClusterDetailsMap(result, true);
+            if (includeCCSMetadata) {
+                mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
+            }
+            assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
+            if (includeCCSMetadata) {
+                assertClusterDetailsMap(result, true);
+            }
         }
     }
 
     public void testUngroupedAggs() throws Exception {
         {
-            Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data)");
+            boolean includeCCSMetadata = randomBoolean();
+            Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data)", includeCCSMetadata);
             var columns = List.of(Map.of("name", "total", "type", "long"));
             long sum = Stream.concat(localDocs.stream(), remoteDocs.stream()).mapToLong(d -> d.data).sum();
             var values = List.of(List.of(Math.toIntExact(sum)));
 
             // check all sections of map except _cluster/details
             MapMatcher mapMatcher = matchesMap();
-            assertMap(
-                result,
-                mapMatcher.entry("columns", columns)
-                    .entry("values", values)
-                    .entry("took", greaterThanOrEqualTo(0))
-                    .entry("_clusters", any(Map.class))
-            );
-            assertClusterDetailsMap(result, false);
+            if (includeCCSMetadata) {
+                mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
+            }
+            assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
+            if (includeCCSMetadata) {
+                assertClusterDetailsMap(result, false);
+            }
         }
         {
-            Map<String, Object> result = run("FROM *:test-remote-index | STATS total = SUM(data)");
+            boolean includeCCSMetadata = randomBoolean();
+            Map<String, Object> result = run("FROM *:test-remote-index | STATS total = SUM(data)", includeCCSMetadata);
+            var columns = List.of(Map.of("name", "total", "type", "long"));
+            long sum = remoteDocs.stream().mapToLong(d -> d.data).sum();
+            var values = List.of(List.of(Math.toIntExact(sum)));
+
+            MapMatcher mapMatcher = matchesMap();
+            if (includeCCSMetadata) {
+                mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
+            }
+            assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
+            if (includeCCSMetadata) {
+                assertClusterDetailsMap(result, true);
+            }
+        }
+        {
+            Map<String, Object> result = runWithColumnarAndIncludeCCSMetadata("FROM *:test-remote-index | STATS total = SUM(data)");
             var columns = List.of(Map.of("name", "total", "type", "long"));
             long sum = remoteDocs.stream().mapToLong(d -> d.data).sum();
             var values = List.of(List.of(Math.toIntExact(sum)));
 
-            // check all sections of map except _cluster/details
             MapMatcher mapMatcher = matchesMap();
             assertMap(
                 result,
@@ -269,7 +294,11 @@ private void assertClusterDetailsMap(Map<String, Object> result, boolean remoteO
 
     public void testGroupedAggs() throws Exception {
         {
-            Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data) BY color | SORT color");
+            boolean includeCCSMetadata = randomBoolean();
+            Map<String, Object> result = run(
+                "FROM test-local-index,*:test-remote-index | STATS total = SUM(data) BY color | SORT color",
+                includeCCSMetadata
+            );
             var columns = List.of(Map.of("name", "total", "type", "long"), Map.of("name", "color", "type", "keyword"));
             var values = Stream.concat(localDocs.stream(), remoteDocs.stream())
                 .collect(Collectors.toMap(d -> d.color, Doc::data, Long::sum))
@@ -280,17 +309,20 @@ public void testGroupedAggs() throws Exception {
                 .toList();
 
             MapMatcher mapMatcher = matchesMap();
-            assertMap(
-                result,
-                mapMatcher.entry("columns", columns)
-                    .entry("values", values)
-                    .entry("took", greaterThanOrEqualTo(0))
-                    .entry("_clusters", any(Map.class))
-            );
-            assertClusterDetailsMap(result, false);
+            if (includeCCSMetadata) {
+                mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
+            }
+            assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
+            if (includeCCSMetadata) {
+                assertClusterDetailsMap(result, false);
+            }
         }
         {
-            Map<String, Object> result = run("FROM *:test-remote-index | STATS total = SUM(data) by color | SORT color");
+            boolean includeCCSMetadata = randomBoolean();
+            Map<String, Object> result = run(
+                "FROM *:test-remote-index | STATS total = SUM(data) by color | SORT color",
+                includeCCSMetadata
+            );
             var columns = List.of(Map.of("name", "total", "type", "long"), Map.of("name", "color", "type", "keyword"));
             var values = remoteDocs.stream()
                 .collect(Collectors.toMap(d -> d.color, Doc::data, Long::sum))
@@ -300,16 +332,15 @@ public void testGroupedAggs() throws Exception {
                 .map(e -> List.of(Math.toIntExact(e.getValue()), e.getKey()))
                 .toList();
 
-            // check all sections of map except _cluster/details
+            // check all sections of map except _clusters/details
             MapMatcher mapMatcher = matchesMap();
-            assertMap(
-                result,
-                mapMatcher.entry("columns", columns)
-                    .entry("values", values)
-                    .entry("took", greaterThanOrEqualTo(0))
-                    .entry("_clusters", any(Map.class))
-            );
-            assertClusterDetailsMap(result, true);
+            if (includeCCSMetadata) {
+                mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
+            }
+            assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
+            if (includeCCSMetadata) {
+                assertClusterDetailsMap(result, true);
+            }
         }
     }
 
diff --git a/x-pack/plugin/esql/qa/server/multi-node/build.gradle b/x-pack/plugin/esql/qa/server/multi-node/build.gradle
index 9f8ca78aba81e..2dcc001c4e159 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/build.gradle
+++ b/x-pack/plugin/esql/qa/server/multi-node/build.gradle
@@ -11,6 +11,7 @@ dependencies {
 
   clusterPlugins project(':plugins:mapper-size')
   clusterPlugins project(':plugins:mapper-murmur3')
+  clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
 }
 
 GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest")
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java
index bda10709ed947..64c113345bd53 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java
@@ -14,7 +14,7 @@
 
 public class EsqlSpecIT extends EsqlSpecTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> {});
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
 
     @Override
     protected String getTestRestCluster() {
diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle
index ab8e3d4b32d9a..a37db5dc245e0 100644
--- a/x-pack/plugin/esql/qa/server/single-node/build.gradle
+++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle
@@ -22,6 +22,7 @@ dependencies {
 
   clusterPlugins project(':plugins:mapper-size')
   clusterPlugins project(':plugins:mapper-murmur3')
+  clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
 }
 
 restResources {
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java
index 676fffd553ca8..368eebe808eee 100644
--- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java
@@ -18,7 +18,7 @@
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
 public class EsqlSpecIT extends EsqlSpecTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = Clusters.testCluster();
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
 
     @Override
     protected String getTestRestCluster() {
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
index 3388f6f517bdf..7de4ee4ccae28 100644
--- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
@@ -76,6 +76,7 @@ public void testBasicEsql() throws IOException {
         indexTimestampData(1);
 
         RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | stats avg(value)");
+        requestObjectBuilder().includeCCSMetadata(randomBoolean());
         if (Build.current().isSnapshot()) {
             builder.pragmas(Settings.builder().put("data_partitioning", "shard").build());
         }
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
index 319e67512c7ac..57f58fc448822 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
@@ -65,7 +65,10 @@
 import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues;
-import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET_MAP;
+import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.availableDatasetsForEs;
+import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.clusterHasInferenceEndpoint;
+import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.createInferenceEndpoint;
+import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteInferenceEndpoint;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
 
@@ -129,7 +132,11 @@ protected EsqlSpecTestCase(
 
     @Before
     public void setup() throws IOException {
-        if (indexExists(CSV_DATASET_MAP.keySet().iterator().next()) == false) {
+        if (supportsInferenceTestService() && clusterHasInferenceEndpoint(client()) == false) {
+            createInferenceEndpoint(client());
+        }
+
+        if (indexExists(availableDatasetsForEs(client()).iterator().next().indexName()) == false) {
             loadDataSetIntoEs(client());
         }
     }
@@ -148,6 +155,8 @@ public static void wipeTestData() throws IOException {
                 throw e;
             }
         }
+
+        deleteInferenceEndpoint(client());
     }
 
     public boolean logResults() {
@@ -164,6 +173,9 @@ public final void test() throws Throwable {
     }
 
     protected void shouldSkipTest(String testName) throws IOException {
+        if (testCase.requiredCapabilities.contains("semantic_text_type")) {
+            assumeTrue("Inference test service needs to be supported for semantic_text", supportsInferenceTestService());
+        }
         checkCapabilities(adminClient(), testFeatureService, testName, testCase);
         assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, instructions, Version.CURRENT));
     }
@@ -207,6 +219,10 @@ protected static void checkCapabilities(RestClient client, TestFeatureService te
         }
     }
 
+    protected boolean supportsInferenceTestService() {
+        return true;
+    }
+
     protected final void doTest() throws Throwable {
         RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values()));
 
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
index 8163e73078c71..8c52a24231a41 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -128,6 +129,7 @@ public static class RequestObjectBuilder {
         private Boolean keepOnCompletion = null;
 
         private Boolean profile = null;
+        private Boolean includeCCSMetadata = null;
 
         private CheckedConsumer<XContentBuilder, IOException> filter;
 
@@ -197,6 +199,11 @@ public RequestObjectBuilder profile(boolean profile) {
             return this;
         }
 
+        public RequestObjectBuilder includeCCSMetadata(boolean includeCCSMetadata) {
+            this.includeCCSMetadata = includeCCSMetadata;
+            return this;
+        }
+
         public RequestObjectBuilder filter(CheckedConsumer<XContentBuilder, IOException> filter) {
             this.filter = filter;
             return this;
@@ -220,6 +227,9 @@ public RequestObjectBuilder build() throws IOException {
                 if (profile != null) {
                     builder.field("profile", profile);
                 }
+                if (includeCCSMetadata != null) {
+                    builder.field("include_ccs_metadata", includeCCSMetadata);
+                }
                 if (filter != null) {
                     builder.startObject("filter");
                     filter.accept(builder);
@@ -388,9 +398,8 @@ public void testOutOfRangeComparisons() throws IOException {
             "long",
             // TODO: https://github.com/elastic/elasticsearch/issues/102935
             // "unsigned_long",
-            // TODO: https://github.com/elastic/elasticsearch/issues/100130
-            // "half_float",
-            // "float",
+            "half_float",
+            "float",
             "double",
             "scaled_float"
         );
@@ -661,6 +670,10 @@ public void testErrorMessageForArrayValuesInParams() throws IOException {
     }
 
     public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOException {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         bulkLoadTestData(10);
         // positive
         var query = requestObjectBuilder().query(
@@ -671,12 +684,9 @@ public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOExcepti
             )
         )
             .params(
-                "[{\"n1\" : {\"value\" : \"integer\" , \"kind\" : \"identifier\"}},"
-                    + "{\"n2\" : {\"value\" : \"short\" , \"kind\" : \"identifier\"}}, "
-                    + "{\"n3\" : {\"value\" : \"double\" , \"kind\" : \"identifier\"}},"
-                    + "{\"n4\" : {\"value\" : \"boolean\" , \"kind\" : \"identifier\"}}, "
-                    + "{\"n5\" : {\"value\" : \"xx*\" , \"kind\" : \"pattern\"}}, "
-                    + "{\"fn1\" : {\"value\" : \"max\" , \"kind\" : \"identifier\"}}]"
+                "[{\"n1\" : {\"identifier\" : \"integer\"}}, {\"n2\" : {\"identifier\" : \"short\"}}, "
+                    + "{\"n3\" : {\"identifier\" : \"double\"}}, {\"n4\" : {\"identifier\" : \"boolean\"}}, "
+                    + "{\"n5\" : {\"pattern\" : \"xx*\"}}, {\"fn1\" : {\"identifier\" : \"max\"}}]"
             );
         Map<String, Object> result = runEsql(query);
         Map<String, String> colA = Map.of("name", "boolean", "type", "boolean");
@@ -715,10 +725,7 @@ public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOExcepti
                 ResponseException.class,
                 () -> runEsqlSync(
                     requestObjectBuilder().query(format(null, "from {} | {}", testIndexName(), command.getKey()))
-                        .params(
-                            "[{\"n1\" : {\"value\" : \"integer\" , \"kind\" : \"identifier\"}},"
-                                + "{\"n2\" : {\"value\" : \"short\" , \"kind\" : \"identifier\"}}]"
-                        )
+                        .params("[{\"n1\" : {\"identifier\" : \"integer\"}}, {\"n2\" : {\"identifier\" : \"short\"}}]")
                 )
             );
             error = re.getMessage();
@@ -738,9 +745,8 @@ public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOExcepti
                 () -> runEsqlSync(
                     requestObjectBuilder().query(format(null, "from {} | {}", testIndexName(), command.getKey()))
                         .params(
-                            "[{\"n1\" : {\"value\" : \"`n1`\" , \"kind\" : \"identifier\"}},"
-                                + "{\"n2\" : {\"value\" : \"`n2`\" , \"kind\" : \"identifier\"}}, "
-                                + "{\"n3\" : {\"value\" : \"`n3`\" , \"kind\" : \"identifier\"}}]"
+                            "[{\"n1\" : {\"identifier\" : \"`n1`\"}}, {\"n2\" : {\"identifier\" : \"`n2`\"}}, "
+                                + "{\"n3\" : {\"identifier\" : \"`n3`\"}}]"
                         )
                 )
             );
@@ -768,7 +774,7 @@ public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOExcepti
                 ResponseException.class,
                 () -> runEsqlSync(
                     requestObjectBuilder().query(format(null, "from {} | ?cmd {}", testIndexName(), command.getValue()))
-                        .params("[{\"cmd\" : {\"value\" : \"" + command.getKey() + "\", \"kind\" : \"identifier\"}}]")
+                        .params("[{\"cmd\" : {\"identifier\" : \"" + command.getKey() + "\"}}]")
                 )
             );
             error = re.getMessage();
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java
index 36d785c24ab23..1a2aa122c85ca 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java
@@ -134,7 +134,11 @@ private static void assertMetadata(
                         || expectedType == UNSIGNED_LONG)) {
                     continue;
                 }
-                if (blockType == Type.KEYWORD && (expectedType == Type.IP || expectedType == Type.VERSION || expectedType == Type.TEXT)) {
+                if (blockType == Type.KEYWORD
+                    && (expectedType == Type.IP
+                        || expectedType == Type.VERSION
+                        || expectedType == Type.TEXT
+                        || expectedType == Type.SEMANTIC_TEXT)) {
                     // Type.asType translates all bytes references into keywords
                     continue;
                 }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
index eaec6811fbc24..bd8bd0f688837 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
@@ -447,6 +447,7 @@ public enum Type {
         SCALED_FLOAT(s -> s == null ? null : scaledFloat(s, "100"), Double.class),
         KEYWORD(Object::toString, BytesRef.class),
         TEXT(Object::toString, BytesRef.class),
+        SEMANTIC_TEXT(Object::toString, BytesRef.class),
         IP(
             StringUtils::parseIP,
             (l, r) -> l instanceof String maybeIP
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
index d63585086f1cd..cf9d66727a900 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
@@ -19,6 +19,7 @@
 import org.apache.logging.log4j.core.config.plugins.util.PluginManager;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.Strings;
@@ -36,9 +37,11 @@
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.COMMA_ESCAPING_REGEX;
@@ -81,6 +84,7 @@ public class CsvTestsDataLoader {
     private static final TestsDataset K8S = new TestsDataset("k8s", "k8s-mappings.json", "k8s.csv").withSetting("k8s-settings.json");
     private static final TestsDataset ADDRESSES = new TestsDataset("addresses");
     private static final TestsDataset BOOKS = new TestsDataset("books");
+    private static final TestsDataset SEMANTIC_TEXT = new TestsDataset("semantic_text").withInferenceEndpoint(true);
 
     public static final Map<String, TestsDataset> CSV_DATASET_MAP = Map.ofEntries(
         Map.entry(EMPLOYEES.indexName, EMPLOYEES),
@@ -112,7 +116,8 @@ public class CsvTestsDataLoader {
         Map.entry(K8S.indexName, K8S),
         Map.entry(DISTANCES.indexName, DISTANCES),
         Map.entry(ADDRESSES.indexName, ADDRESSES),
-        Map.entry(BOOKS.indexName, BOOKS)
+        Map.entry(BOOKS.indexName, BOOKS),
+        Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT)
     );
 
     private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");
@@ -219,8 +224,13 @@ public static void main(String[] args) throws IOException {
         }
     }
 
-    private static void loadDataSetIntoEs(RestClient client, IndexCreator indexCreator) throws IOException {
-        loadDataSetIntoEs(client, LogManager.getLogger(CsvTestsDataLoader.class), indexCreator);
+    public static Set<TestsDataset> availableDatasetsForEs(RestClient client) throws IOException {
+        boolean inferenceEnabled = clusterHasInferenceEndpoint(client);
+
+        return CSV_DATASET_MAP.values()
+            .stream()
+            .filter(d -> d.requiresInferenceEndpoint == false || inferenceEnabled)
+            .collect(Collectors.toCollection(HashSet::new));
     }
 
     public static void loadDataSetIntoEs(RestClient client) throws IOException {
@@ -229,22 +239,61 @@ public static void loadDataSetIntoEs(RestClient client) throws IOException {
         });
     }
 
-    public static void loadDataSetIntoEs(RestClient client, Logger logger) throws IOException {
-        loadDataSetIntoEs(client, logger, (restClient, indexName, indexMapping, indexSettings) -> {
-            ESRestTestCase.createIndex(restClient, indexName, indexSettings, indexMapping, null);
-        });
-    }
+    private static void loadDataSetIntoEs(RestClient client, IndexCreator indexCreator) throws IOException {
+        Logger logger = LogManager.getLogger(CsvTestsDataLoader.class);
 
-    private static void loadDataSetIntoEs(RestClient client, Logger logger, IndexCreator indexCreator) throws IOException {
-        for (var dataset : CSV_DATASET_MAP.values()) {
+        Set<String> loadedDatasets = new HashSet<>();
+        for (var dataset : availableDatasetsForEs(client)) {
             load(client, dataset, logger, indexCreator);
+            loadedDatasets.add(dataset.indexName);
         }
-        forceMerge(client, CSV_DATASET_MAP.keySet(), logger);
+        forceMerge(client, loadedDatasets, logger);
         for (var policy : ENRICH_POLICIES) {
             loadEnrichPolicy(client, policy.policyName, policy.policyFileName, logger);
         }
     }
 
+    /** The semantic_text mapping type require an inference endpoint that needs to be setup before creating the index. */
+    public static void createInferenceEndpoint(RestClient client) throws IOException {
+        Request request = new Request("PUT", "_inference/sparse_embedding/test_sparse_inference");
+        request.setJsonEntity("""
+                  {
+                   "service": "test_service",
+                   "service_settings": {
+                     "model": "my_model",
+                     "api_key": "abc64"
+                   },
+                   "task_settings": {
+                   }
+                 }
+            """);
+        client.performRequest(request);
+    }
+
+    public static void deleteInferenceEndpoint(RestClient client) throws IOException {
+        try {
+            client.performRequest(new Request("DELETE", "_inference/test_sparse_inference"));
+        } catch (ResponseException e) {
+            // 404 here means the endpoint was not created
+            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
+                throw e;
+            }
+        }
+    }
+
+    public static boolean clusterHasInferenceEndpoint(RestClient client) throws IOException {
+        Request request = new Request("GET", "_inference/sparse_embedding/test_sparse_inference");
+        try {
+            client.performRequest(request);
+        } catch (ResponseException e) {
+            if (e.getResponse().getStatusLine().getStatusCode() == 404) {
+                return false;
+            }
+            throw e;
+        }
+        return true;
+    }
+
     private static void loadEnrichPolicy(RestClient client, String policyName, String policyFileName, Logger logger) throws IOException {
         URL policyMapping = CsvTestsDataLoader.class.getResource("/" + policyFileName);
         if (policyMapping == null) {
@@ -511,34 +560,79 @@ public record TestsDataset(
         String dataFileName,
         String settingFileName,
         boolean allowSubFields,
-        Map<String, String> typeMapping
+        Map<String, String> typeMapping,
+        boolean requiresInferenceEndpoint
     ) {
         public TestsDataset(String indexName, String mappingFileName, String dataFileName) {
-            this(indexName, mappingFileName, dataFileName, null, true, null);
+            this(indexName, mappingFileName, dataFileName, null, true, null, false);
         }
 
         public TestsDataset(String indexName) {
-            this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null);
+            this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, false);
         }
 
         public TestsDataset withIndex(String indexName) {
-            return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping);
+            return new TestsDataset(
+                indexName,
+                mappingFileName,
+                dataFileName,
+                settingFileName,
+                allowSubFields,
+                typeMapping,
+                requiresInferenceEndpoint
+            );
         }
 
         public TestsDataset withData(String dataFileName) {
-            return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping);
+            return new TestsDataset(
+                indexName,
+                mappingFileName,
+                dataFileName,
+                settingFileName,
+                allowSubFields,
+                typeMapping,
+                requiresInferenceEndpoint
+            );
         }
 
         public TestsDataset withSetting(String settingFileName) {
-            return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping);
+            return new TestsDataset(
+                indexName,
+                mappingFileName,
+                dataFileName,
+                settingFileName,
+                allowSubFields,
+                typeMapping,
+                requiresInferenceEndpoint
+            );
         }
 
         public TestsDataset noSubfields() {
-            return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, false, typeMapping);
+            return new TestsDataset(
+                indexName,
+                mappingFileName,
+                dataFileName,
+                settingFileName,
+                false,
+                typeMapping,
+                requiresInferenceEndpoint
+            );
         }
 
         public TestsDataset withTypeMapping(Map<String, String> typeMapping) {
-            return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping);
+            return new TestsDataset(
+                indexName,
+                mappingFileName,
+                dataFileName,
+                settingFileName,
+                allowSubFields,
+                typeMapping,
+                requiresInferenceEndpoint
+            );
+        }
+
+        public TestsDataset withInferenceEndpoint(boolean needsInference) {
+            return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference);
         }
     }
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
index 693b6fa8bd670..d71c66b4c467f 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.DateUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
@@ -260,7 +261,7 @@ public boolean isIndexed(String field) {
 
     public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY));
 
-    public static final Verifier TEST_VERIFIER = new Verifier(new Metrics());
+    public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry()));
 
     private EsqlTestUtils() {}
 
@@ -647,7 +648,7 @@ public static Literal randomLiteral(DataType type) {
             case KEYWORD -> new BytesRef(randomAlphaOfLength(5));
             case IP -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())));
             case TIME_DURATION -> Duration.ofMillis(randomLongBetween(-604800000L, 604800000L)); // plus/minus 7 days
-            case TEXT -> new BytesRef(randomAlphaOfLength(50));
+            case TEXT, SEMANTIC_TEXT -> new BytesRef(randomAlphaOfLength(50));
             case VERSION -> randomVersion().toBytesRef();
             case GEO_POINT -> GEO.asWkb(GeometryTestUtils.randomPoint());
             case CARTESIAN_POINT -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint());
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
index 1fdb6150a0e81..237c6a9af197f 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
@@ -367,9 +367,9 @@ date1:date               | dd_ms:integer
 2023-12-02T11:00:00.000Z | 1000
 ;
 
-evalDateDiffMonthAsWhole0Months
+evalDateDiffMonthAsWhole0Months#[skip:-8.14.1, reason:omitting millis/timezone not allowed before 8.14]
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00:00:00")
 | EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to)
 ;
 
@@ -378,9 +378,9 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIM
 
 ;
 
-evalDateDiffMonthAsWhole1Month
+evalDateDiffMonthAsWhole1Month#[skip:-8.14.1, reason:omitting millis/timezone not allowed before 8.14]
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-02-01T00:00:00")
 | EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to)
 ;
 
@@ -392,7 +392,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIM
 evalDateDiffYearAsWhole0Years
 required_capability: date_diff_year_calendarial
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00:00:00")
 | EVAL msecs=DATE_DIFF("milliseconds", from, to), years=DATE_DIFF("year", from, to)
 ;
 
@@ -403,7 +403,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIM
 evalDateDiffYearAsWhole1Year
 required_capability: date_diff_year_calendarial
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2025-01-01T00:00:00")
 | EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to)
 ;
 
@@ -414,7 +414,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIM
 evalDateDiffYearAsWhole1Year
 required_capability: date_diff_year_calendarial
 
-ROW from="2024-01-01T00:00:00Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2024-01-01T00:00:00Z"), to=TO_DATETIME("2025-01-01T00:00:00")
 | EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to)
 ;
 
@@ -426,9 +426,9 @@ evalDateDiffYearForDocs
 required_capability: date_diff_year_calendarial
 
 // tag::evalDateDiffYearForDocs[]
-ROW end_23="2023-12-31T23:59:59.999Z"::DATETIME,
-  start_24="2024-01-01T00:00:00.000Z"::DATETIME,
-    end_24="2024-12-31T23:59:59.999"::DATETIME
+ROW end_23=TO_DATETIME("2023-12-31T23:59:59.999Z"),
+  start_24=TO_DATETIME("2024-01-01T00:00:00.000Z"),
+    end_24=TO_DATETIME("2024-12-31T23:59:59.999")
 | EVAL end23_to_start24=DATE_DIFF("year", end_23, start_24)
 | EVAL end23_to_end24=DATE_DIFF("year", end_23, end_24)
 | EVAL start_to_end_24=DATE_DIFF("year", start_24, end_24)
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv
index 83a2f3cb1c281..26b6f055221a6 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv
@@ -6,4 +6,6 @@ millis:date,nanos:date_nanos,num:long
 2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z,1698068014937193000
 2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z,1698064048948000000
 2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z,1698063303360103847
+2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z,1698063303360103847
 1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z], 0
+1999-10-22T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z], 0
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
index ad7149b0f742f..d0edc1f07d021 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
@@ -38,9 +38,10 @@ nanos:date_nanos
 mv_min on date nanos
 required_capability: date_nanos_type
 
-FROM date_nanos | SORT millis ASC | EVAL nanos = MV_MIN(nanos) | KEEP nanos | LIMIT 1;
+FROM date_nanos | SORT millis ASC | WHERE millis < "2000-01-01" | EVAL nanos = MV_MIN(nanos) |  KEEP nanos; 
 
 nanos:date_nanos
+2023-03-23T12:15:03.360103847Z
 2023-01-23T13:55:01.543123456Z
 ;
 
@@ -56,9 +57,10 @@ ct:integer
 mv_first on date nanos
 required_capability: date_nanos_type
 
-FROM date_nanos | SORT millis ASC | EVAL nanos = MV_FIRST(nanos) | KEEP nanos | LIMIT 1;
+FROM date_nanos | SORT millis ASC | WHERE millis < "2000-01-01" | EVAL nanos = MV_FIRST(nanos) |  KEEP nanos;
 
 nanos:date_nanos
+2023-03-23T12:15:03.360103847Z
 2023-01-23T13:55:01.543123456Z
 ;
 
@@ -214,6 +216,7 @@ l:long
 1698068014937193000
 1698064048948000000
 1698063303360103847
+1698063303360103847
 ;
 
 long to date nanos, index version
@@ -229,6 +232,7 @@ d:date_nanos
 2023-10-23T13:33:34.937193000Z
 2023-10-23T12:27:28.948000000Z
 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360103847Z
 ;
 
 date_nanos to date nanos, index version
@@ -244,6 +248,7 @@ d:date_nanos
 2023-10-23T13:33:34.937193000Z
 2023-10-23T12:27:28.948000000Z
 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360103847Z
 ;
 
 attempt to cast the result of a fold to date nanos
@@ -263,3 +268,97 @@ ROW a = TO_DATE_NANOS(null), b = TO_DATE_NANOS(null + 1::long), c = TO_DATE_NANO
 a:date_nanos | b:date_nanos | c:date_nanos
 null         | null         | null
 ;
+
+Coalasce date nanos
+required_capability: to_date_nanos
+
+ROW a = COALESCE(null, TO_DATE_NANOS(1698069301543123456));
+
+a:date_nanos
+2023-10-23T13:55:01.543123456Z
+;
+
+Case date nanos result
+required_capability: to_date_nanos
+
+ROW a = CASE(false, TO_DATE_NANOS(0::long), TO_DATE_NANOS(1698069301543123456));
+
+a:date_nanos
+2023-10-23T13:55:01.543123456Z
+;
+
+Greatest date nanos
+required_capability: least_greatest_for_datenanos
+
+ROW a = GREATEST(TO_DATE_NANOS("2023-10-23T13:55:01.543123456"), TO_DATE_NANOS("2023-10-23T13:53:55.832987654"));
+
+a:date_nanos
+2023-10-23T13:55:01.543123456Z
+;
+
+Least date nanos
+required_capability: least_greatest_for_datenanos
+
+ROW a = LEAST(TO_DATE_NANOS("2023-10-23T13:55:01.543123456"), TO_DATE_NANOS("2023-10-23T13:53:55.832987654"));
+
+a:date_nanos
+2023-10-23T13:53:55.832987654Z
+;
+
+mv_dedup over date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | WHERE millis < "2000-01-01" | EVAL a = MV_DEDUPE(nanos) | SORT millis DESC | KEEP a;
+
+a:date_nanos
+[2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193000Z, 2023-03-23T12:15:03.360103847Z]
+2023-03-23T12:15:03.360103847Z
+;
+
+mv_sort over date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | WHERE millis < "2000-01-01" | EVAL a = MV_SORT(nanos, "asc") | SORT millis DESC | KEEP a;
+
+a:date_nanos
+[2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193000Z, 2023-03-23T12:15:03.360103847Z]
+[2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z]
+;
+
+mv_slice over date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | WHERE millis < "2000-01-01" | EVAL a = MV_SLICE(MV_SORT(nanos, "asc"), 1, 2) | SORT millis DESC | KEEP a;
+
+a:date_nanos
+[2023-02-23T13:33:34.937193000Z, 2023-03-23T12:15:03.360103847Z]
+[2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z]
+;
+
+
+Max and Min of date nanos
+required_capability: date_nanos_aggregations
+
+FROM date_nanos | STATS max = MAX(nanos), min = MIN(nanos);
+
+max:date_nanos                 | min:date_nanos
+2023-10-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543123456Z
+;
+
+Count and count distinct of date nanos
+required_capability: date_nanos_aggregations
+
+FROM date_nanos | WHERE millis > "2020-01-01" | STATS count = COUNT(nanos), count_distinct = COUNT_DISTINCT(nanos);
+
+count:long | count_distinct:long
+8          | 7
+;
+
+Values aggregation on date nanos
+required_capability: date_nanos_aggregations
+
+FROM date_nanos | WHERE millis > "2020-01-01" | STATS v = MV_SORT(VALUES(nanos), "DESC");
+
+v:date_nanos
+[2023-10-23T13:55:01.543123456Z, 2023-10-23T13:53:55.832987654Z, 2023-10-23T13:52:55.015787878Z, 2023-10-23T13:51:54.732102837Z, 2023-10-23T13:33:34.937193000Z, 2023-10-23T12:27:28.948000000Z, 2023-10-23T12:15:03.360103847Z]
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec
index 15fe6853ae491..a9c5a5214f159 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec
@@ -382,7 +382,7 @@ count:long | languages:integer
 basicGrok
 // tag::basicGrok[]
 ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" 
-| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}" 
+| GROK a """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}""" 
 | KEEP date, ip, email, num
 // end::basicGrok[]
 ;
@@ -396,7 +396,7 @@ date:keyword          | ip:keyword    | email:keyword       | num:keyword
 grokWithConversionSuffix
 // tag::grokWithConversionSuffix[]
 ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" 
-| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" 
+| GROK a """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}""" 
 | KEEP date, ip, email, num
 // end::grokWithConversionSuffix[]
 ;
@@ -410,7 +410,7 @@ date:keyword              | ip:keyword    | email:keyword       | num:integer
 grokWithToDatetime
 // tag::grokWithToDatetime[]
 ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" 
-| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" 
+| GROK a """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}""" 
 | KEEP date, ip, email, num
 | EVAL date = TO_DATETIME(date)
 // end::grokWithToDatetime[]
@@ -436,11 +436,27 @@ ROW a = "1.2.3.4 [2023-01-23T12:15:00.000Z] Connected"
 // end::grokWithEscape-result[]
 ;
 
+
+grokWithEscapeTripleQuotes
+// tag::grokWithEscapeTripleQuotes[]
+ROW a = "1.2.3.4 [2023-01-23T12:15:00.000Z] Connected"
+| GROK a """%{IP:ip} \[%{TIMESTAMP_ISO8601:@timestamp}\] %{GREEDYDATA:status}"""
+// end::grokWithEscapeTripleQuotes[]
+| KEEP @timestamp
+;
+
+// tag::grokWithEscapeTripleQuotes-result[]
+@timestamp:keyword
+2023-01-23T12:15:00.000Z
+// end::grokWithEscapeTripleQuotes-result[]
+;
+
+
 grokWithDuplicateFieldNames
 // tag::grokWithDuplicateFieldNames[]
 FROM addresses
 | KEEP city.name, zip_code
-| GROK zip_code "%{WORD:zip_parts} %{WORD:zip_parts}"
+| GROK zip_code """%{WORD:zip_parts} %{WORD:zip_parts}"""
 // end::grokWithDuplicateFieldNames[]
 | SORT city.name
 ;
@@ -456,7 +472,7 @@ Tokyo             | 100-7014         | null
 basicDissect
 // tag::basicDissect[]
 ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" 
-| DISSECT a "%{date} - %{msg} - %{ip}"
+| DISSECT a """%{date} - %{msg} - %{ip}"""
 | KEEP date, msg, ip
 // end::basicDissect[]
 ;
@@ -470,7 +486,7 @@ date:keyword             | msg:keyword  | ip:keyword
 dissectWithToDatetime
 // tag::dissectWithToDatetime[]
 ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" 
-| DISSECT a "%{date} - %{msg} - %{ip}" 
+| DISSECT a """%{date} - %{msg} - %{ip}""" 
 | KEEP date, msg, ip
 | EVAL date = TO_DATETIME(date)
 // end::dissectWithToDatetime[]
@@ -485,7 +501,7 @@ some text    | 127.0.0.1  | 2023-01-23T12:15:00.000Z
 dissectRightPaddingModifier
 // tag::dissectRightPaddingModifier[]
 ROW message="1998-08-10T17:15:42          WARN"
-| DISSECT message "%{ts->} %{level}"
+| DISSECT message """%{ts->} %{level}"""
 // end::dissectRightPaddingModifier[]
 ;
 
@@ -498,7 +514,7 @@ message:keyword  | ts:keyword | level:keyword
 dissectEmptyRightPaddingModifier#[skip:-8.11.2, reason:Support for empty right padding modifiers introduced in 8.11.2]
 // tag::dissectEmptyRightPaddingModifier[]
 ROW message="[1998-08-10T17:15:42]          [WARN]"
-| DISSECT message "[%{ts}]%{->}[%{level}]"
+| DISSECT message """[%{ts}]%{->}[%{level}]"""
 // end::dissectEmptyRightPaddingModifier[]
 ;
 
@@ -511,7 +527,7 @@ ROW message="[1998-08-10T17:15:42]          [WARN]"
 dissectAppendModifier
 // tag::dissectAppendModifier[]
 ROW message="john jacob jingleheimer schmidt"
-| DISSECT message "%{+name} %{+name} %{+name} %{+name}" APPEND_SEPARATOR=" "
+| DISSECT message """%{+name} %{+name} %{+name} %{+name}""" APPEND_SEPARATOR=" "
 // end::dissectAppendModifier[]
 ;
 
@@ -524,7 +540,7 @@ john jacob jingleheimer schmidt|john jacob jingleheimer schmidt
 dissectAppendWithOrderModifier
 // tag::dissectAppendWithOrderModifier[]
 ROW message="john jacob jingleheimer schmidt"
-| DISSECT message "%{+name/2} %{+name/4} %{+name/3} %{+name/1}" APPEND_SEPARATOR=","
+| DISSECT message """%{+name/2} %{+name/4} %{+name/3} %{+name/1}""" APPEND_SEPARATOR=","
 // end::dissectAppendWithOrderModifier[]
 ;
 
@@ -537,7 +553,7 @@ john jacob jingleheimer schmidt|schmidt,john,jingleheimer,jacob
 dissectNamedSkipKey
 // tag::dissectNamedSkipKey[]
 ROW message="1.2.3.4 - - 30/Apr/1998:22:00:52 +0000"
-| DISSECT message "%{clientip} %{?ident} %{?auth} %{@timestamp}"
+| DISSECT message """%{clientip} %{?ident} %{?auth} %{@timestamp}"""
 // end::dissectNamedSkipKey[]
 ;
 
@@ -550,7 +566,7 @@ message:keyword  | clientip:keyword  | @timestamp:keyword
 docsLike
 // tag::like[]
 FROM employees
-| WHERE first_name LIKE "?b*"
+| WHERE first_name LIKE """?b*"""
 | KEEP first_name, last_name
 // end::like[]
 | SORT first_name
@@ -566,7 +582,7 @@ Eberhardt      |Terkki
 docsRlike
 // tag::rlike[]
 FROM employees
-| WHERE first_name RLIKE ".leja.*"
+| WHERE first_name RLIKE """.leja.*"""
 | KEEP first_name, last_name
 // end::rlike[]
 ;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json
index 98a3794d977e2..cd572012c6b3a 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json
@@ -12,11 +12,17 @@
     "centroid": {
       "type": "geo_point"
     },
+    "lk": {
+      "type": "keyword"
+    },
     "location": {
       "type": "geo_point"
     },
     "subset": {
       "type": "geo_point"
+    },
+    "disjoint": {
+      "type": "geo_point"
     }
   }
 }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json
new file mode 100644
index 0000000000000..b110d6fd4cdd5
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json
@@ -0,0 +1,73 @@
+{
+  "properties": {
+    "semantic_text_field": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_bool": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_cartesian_point": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_cartesian_shape": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_datetime": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_double": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_geopoint": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_geoshape": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_integer": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_ip": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_long": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_unsigned_long": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_version": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_multi_value": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "st_unicode": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
+    },
+    "host" : {
+      "type" : "keyword"
+    },
+    "description" : {
+      "type" : "text"
+    },
+    "value": {
+      "type": "long"
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec
new file mode 100644
index 0000000000000..b0578aa1a4ed0
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec
@@ -0,0 +1,199 @@
+###############################################
+# Tests for Match function
+#
+
+matchWithField
+required_capability: match_function
+
+// tag::match-with-field[]
+from books 
+| where match(author, "Faulkner")
+| keep book_no, author 
+| sort book_no 
+| limit 5;
+// end::match-with-field[]
+
+// tag::match-with-field-result[]
+book_no:keyword | author:text
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott]
+2713            | William Faulkner
+2847            | Colleen Faulkner
+2883            | William Faulkner
+3293            | Danny Faulkner
+;
+// end::match-with-field-result[]
+
+matchWithMultipleFunctions
+required_capability: match_function
+
+from books 
+| where match(title, "Return") AND match(author, "Tolkien")  
+| keep book_no, title;
+ignoreOrder:true
+
+book_no:keyword | title:text
+2714            | Return of the King Being the Third Part of The Lord of the Rings
+7350            | Return of the Shadow
+;
+
+matchWithQueryExpressions
+required_capability: match_function
+
+from books 
+| where match(title, CONCAT("Return ", " King"))  
+| keep book_no, title;
+ignoreOrder:true
+
+book_no:keyword | title:text
+2714            | Return of the King Being the Third Part of The Lord of the Rings
+7350            | Return of the Shadow
+;
+
+matchAfterKeep
+required_capability: match_function
+
+from books 
+| keep book_no, author 
+| where match(author, "Faulkner")
+| sort book_no 
+| limit 5;
+
+book_no:keyword | author:text
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott]
+2713            | William Faulkner
+2847            | Colleen Faulkner
+2883            | William Faulkner
+3293            | Danny Faulkner
+;
+
+matchAfterDrop
+required_capability: match_function
+
+from books 
+| drop ratings, description, year, publisher, title, author.keyword
+| where match(author, "Faulkner")
+| keep book_no, author
+| sort book_no 
+| limit 5;
+
+book_no:keyword | author:text
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott]
+2713            | William Faulkner
+2847            | Colleen Faulkner
+2883            | William Faulkner
+3293            | Danny Faulkner
+;
+
+matchAfterEval
+required_capability: match_function
+
+from books 
+| eval stars = to_long(ratings / 2.0) 
+| where match(author, "Faulkner")
+| sort book_no 
+| keep book_no, author, stars
+| limit 5;
+
+book_no:keyword | author:text                                           | stars:long
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott]    | 3
+2713            | William Faulkner                                      | 2
+2847            | Colleen Faulkner                                      | 3
+2883            | William Faulkner                                      | 2
+3293            | Danny Faulkner                                        | 2
+;
+
+matchWithConjunction
+required_capability: match_function
+
+from books 
+| where match(title, "Rings") and ratings > 4.6
+| keep book_no, title;
+ignoreOrder:true
+
+book_no:keyword | title:text
+4023            |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings
+7140            |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1)     
+;
+
+matchWithFunctionPushedToLucene
+required_capability: match_function
+
+from hosts 
+| where match(host, "beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") 
+| keep card, host, ip0, ip1;
+ignoreOrder:true
+
+card:keyword   |host:keyword   |ip0:ip                   |ip1:ip
+eth1           |beta           |127.0.0.1                |127.0.0.2
+;
+
+matchWithNonPushableConjunction
+required_capability: match_function
+
+from books 
+| where match(title, "Rings") and length(title) > 75
+| keep book_no, title;
+ignoreOrder:true
+
+book_no:keyword | title:text
+4023            | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings
+;
+
+matchWithMultipleWhereClauses
+required_capability: match_function
+
+from books 
+| where match(title, "rings") 
+| where match(title, "lord") 
+| keep book_no, title;
+ignoreOrder:true
+
+book_no:keyword | title:text
+2675            | The Lord of the Rings - Boxed Set           
+2714            | Return of the King Being the Third Part of The Lord of the Rings
+4023            | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings
+7140            | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1)
+;
+
+matchMultivaluedField
+required_capability: match_function
+
+from employees 
+| where match(job_positions, "Tech Lead") and match(job_positions, "Reporting Analyst") 
+| keep emp_no, first_name, last_name;
+ignoreOrder:true
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10004          | Chirstian          | Koblick        
+10010          | Duangkaew          | Piveteau       
+10011          | Mary               | Sluis          
+10088          | Jungsoon           | Syrzycki       
+10093          | Sailaja            | Desikan        
+10097          | Remzi              | Waschkowski    
+;
+
+testMultiValuedFieldWithConjunction
+required_capability: match_function
+
+from employees 
+| where match(job_positions, "Data Scientist") and match(job_positions, "Support Engineer")
+| keep emp_no, first_name, last_name;
+ignoreOrder:true
+
+emp_no:integer | first_name:keyword | last_name:keyword  
+10043          | Yishay             | Tzvieli      
+;
+
+testMatchAndQueryStringFunctions
+required_capability: match_function
+required_capability: qstr_function
+
+from employees 
+| where match(job_positions, "Data Scientist") and qstr("job_positions: (Support Engineer) and gender: F")
+| keep emp_no, first_name, last_name;
+ignoreOrder:true
+
+emp_no:integer | first_name:keyword | last_name:keyword  
+10041          | Uri                 | Lenart         
+10043          | Yishay              | Tzvieli        
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec
index b00bb5143726c..da069836504d4 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec
@@ -1379,6 +1379,63 @@ d:double | c:double
 -0.0     | -0.0
 ;
 
+hypot
+required_capability: fn_hypot
+// tag::hypot[]
+ROW a = 3.0, b = 4.0
+| EVAL c = HYPOT(a, b)
+// end::hypot[]
+;
+
+// tag::hypot-result[]
+a:double | b:double | c:double
+3.0      | 4.0      | 5.0
+// end::hypot-result[]
+;
+
+hypotWithFrom
+required_capability: fn_hypot
+FROM ul_logs
+| WHERE id > 95
+| EVAL bytes_hypot = HYPOT(bytes_in, bytes_out)
+| SORT id ASC
+| LIMIT 5
+| KEEP id, bytes_in, bytes_out, bytes_hypot
+;
+
+id:integer | bytes_in:unsigned_long | bytes_out:unsigned_long | bytes_hypot:double
+96         | 9932469097722733505    | 14925592145374204307    | 1.792839209932874E19
+97         | 11620953158540412267   | 3809712277266935082     | 1.2229491401875583E19
+98         | 3448205404634246112    | 5409549730889481641     | 6.415087591258227E18
+99         | 1957665857956635540    | 352442273299370793      | 1.9891382977102218E18
+100        | 16462768484251021236   | 15616395223975497926    | 2.2691287886707827E19
+;
+
+hypotBothNull
+required_capability: fn_hypot
+FROM ul_logs
+| WHERE bytes_in IS NULL and bytes_out IS NULL
+| LIMIT 1
+| EVAL bytes_hypot = HYPOT(bytes_in, bytes_out)
+| KEEP bytes_in, bytes_out, bytes_hypot
+;
+
+bytes_in:unsigned_long | bytes_out:unsigned_long | bytes_hypot:double
+null                   | null                    | null
+;
+
+hypotOneNull
+required_capability: fn_hypot
+FROM ul_logs
+| WHERE id == 41
+| EVAL confused_hypot = HYPOT(id, bytes_in)
+| KEEP id, bytes_in, confused_hypot
+;
+
+id:integer | bytes_in:unsigned_long | confused_hypot:double
+41         | null                   | null
+;
+
 least
 // tag::least[]
 ROW a = 10, b = 20
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv
index c5d7c7f4ee305..efb0aaaa29a4b 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv
@@ -1,14 +1,14 @@
-id:l, intersects:boolean, within:boolean, centroid:geo_point, location:geo_point,              subset:geo_point
-0,    true,               true,           "POINT(0.0 5.0)",   ["POINT(5 5)", "POINT(-5 5)"],   "POINT(5 5)"
-1,    true,               true,           "POINT(0.5 0.5)",   ["POINT(0 1)","POINT(1 0)"],     "POINT(0 1)"
-2,    true,               true,           "POINT(9.0 9.0)",   "POINT(9 9)",                    "POINT(9 9)"
-3,    true,               true,           "POINT(0.0 0.0)",   ["POINT(-9 -9)","POINT(9 9)"],   "POINT(-9 -9)"
-4,    true,               false,          "POINT(10.0 10.0)", ["POINT(5 5)", "POINT(15 15)"],  "POINT(5 5)"
-5,    true,               false,          "POINT(5.5 5.5)",   ["POINT(0 0)","POINT(11 11)"],   "POINT(0 0)"
-6,    true,               false,          "POINT(0.0 -5.0)",  ["POINT(-9 -19)","POINT(9 9)"],  "POINT(-9 -19)"
-7,    false,              false,          "POINT(10.0 10.0)", ["POINT(5 15)", "POINT(15 5)"],  "POINT(5 15)"
-8,    false,              false,          "POINT(5.5 5.5)",   ["POINT(0 11)","POINT(11 0)"],   "POINT(0 11)"
-9,    false,              false,          "POINT(19.0 9.0)",  "POINT(19 9)",                   "POINT(19 9)"
-10,   false,              false,          "POINT(5.0 -5.0)",  ["POINT(-9 -19)","POINT(19 9)"], "POINT(-9 -19)"
-11,   true,               false,          "POINT(0.0 0.0)",   ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"],   ["POINT(0 0)", "POINT(55 55)"]
-12,   true,               false,          "POINT(0.0 0.0)",   ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"],   ["POINT(0 0)", "POINT(55 54)"]
+id:l, intersects:boolean, within:boolean, centroid:geo_point, lk:keyword,                                         location:geo_point,                                 subset:geo_point,               disjoint:geo_point
+0,    true,               true,           "POINT(0.0 5.0)",   ["POINT(5 5)", "POINT(-5 5)"],                      ["POINT(5 5)", "POINT(-5 5)"],                      "POINT(5 5)",                   ["POINT(55 55)", "POINT(65 65)"]
+1,    true,               true,           "POINT(0.5 0.5)",   ["POINT(0 1)","POINT(1 0)"],                        ["POINT(0 1)","POINT(1 0)"],                        "POINT(0 1)",                   ["POINT(55 55)", "POINT(65 65)"]
+2,    true,               true,           "POINT(9.0 9.0)",   "POINT(9 9)",                                       "POINT(9 9)",                                       "POINT(9 9)",                   ["POINT(55 55)", "POINT(65 65)"]
+3,    true,               true,           "POINT(0.0 0.0)",   ["POINT(-9 -9)","POINT(9 9)"],                      ["POINT(-9 -9)","POINT(9 9)"],                      "POINT(-9 -9)",                 ["POINT(55 55)", "POINT(65 65)"]
+4,    true,               false,          "POINT(10.0 10.0)", ["POINT(5 5)", "POINT(15 15)"],                     ["POINT(5 5)", "POINT(15 15)"],                     "POINT(5 5)",                   ["POINT(55 55)", "POINT(65 65)"]
+5,    true,               false,          "POINT(5.5 5.5)",   ["POINT(0 0)","POINT(11 11)"],                      ["POINT(0 0)","POINT(11 11)"],                      "POINT(0 0)",                   ["POINT(55 55)", "POINT(65 65)"]
+6,    true,               false,          "POINT(0.0 -5.0)",  ["POINT(-9 -19)","POINT(9 9)"],                     ["POINT(-9 -19)","POINT(9 9)"],                     "POINT(-9 -19)",                ["POINT(55 55)", "POINT(65 65)"]
+7,    false,              false,          "POINT(10.0 10.0)", ["POINT(5 15)", "POINT(15 5)"],                     ["POINT(5 15)", "POINT(15 5)"],                     "POINT(5 15)",                  ["POINT(55 55)", "POINT(65 65)"]
+8,    false,              false,          "POINT(5.5 5.5)",   ["POINT(0 11)","POINT(11 0)"],                      ["POINT(0 11)","POINT(11 0)"],                      "POINT(0 11)",                  ["POINT(55 55)", "POINT(65 65)"]
+9,    false,              false,          "POINT(19.0 9.0)",  "POINT(19 9)",                                      "POINT(19 9)",                                      "POINT(19 9)",                  ["POINT(55 55)", "POINT(65 65)"]
+10,   false,              false,          "POINT(5.0 -5.0)",  ["POINT(-9 -19)","POINT(19 9)"],                    ["POINT(-9 -19)","POINT(19 9)"],                    "POINT(-9 -19)",                ["POINT(55 55)", "POINT(65 65)"]
+11,   true,               false,          "POINT(0.0 0.0)",   ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"],   ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"],   ["POINT(0 0)", "POINT(55 55)"], ["POINT(55 55)", "POINT(65 65)"]
+12,   true,               false,          "POINT(0.0 0.0)",   ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"],   ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"],   ["POINT(0 0)", "POINT(55 54)"], ["POINT(55 55)", "POINT(65 65)"]
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec
index 6a3521c558fa9..ed9434e9241fc 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec
@@ -60,6 +60,92 @@ id:l | intersects:boolean | within:boolean | st_intersects:boolean | st_within:b
 12   | true               | false          | true                  | false             | false               | "POINT(0.0 0.0)"   | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"]
 ;
 
+spatialDistance
+required_capability: st_distance
+required_capability: spatial_distance_supports_multivalues
+
+FROM multivalue_points
+| EVAL distance_origin=ST_DISTANCE(location, TO_GEOPOINT("POINT(0 0)"))
+| EVAL distance_subset=ST_DISTANCE(location, subset)
+| EVAL distance_disjoint=ST_DISTANCE(location, disjoint)
+| EVAL distance_far=ST_DISTANCE(location, TO_GEOPOINT("POINT(55 55)"))
+| KEEP id, distance_origin, distance_subset, distance_disjoint, distance_far, location, subset, disjoint
+| SORT id
+;
+
+id:l | distance_origin:d  | distance_subset:d | distance_disjoint:d | distance_far:d    | location:geo_point                               | subset:geo_point               | disjoint:geo_point
+0    | 785768.2986429982  | 0.0               | 7114305.127148048   | 7114305.127148048 | ["POINT(5 5)", "POINT(-5 5)"]                    | "POINT(5 5)"                   |  ["POINT(55 55)", "POINT(65 65)"]
+1    | 111195.07310665186 | 0.0               | 7775410.295475619   | 7775410.295475619 | ["POINT(0 1)","POINT(1 0)"]                      | "POINT(0 1)"                   |  ["POINT(55 55)", "POINT(65 65)"]
+2    | 1412359.654429245  | 0.0               | 6511042.96320646    | 6511042.96320646  | "POINT(9 9)"                                     | "POINT(9 9)"                   |  ["POINT(55 55)", "POINT(65 65)"]
+3    | 1412359.654429245  | 0.0               | 6511042.96320646    | 6511042.96320646  | ["POINT(-9 -9)","POINT(9 9)"]                    | "POINT(-9 -9)"                 |  ["POINT(55 55)", "POINT(65 65)"]
+4    | 785768.2986429982  | 0.0               | 5612483.563947934   | 5612483.563947934 | ["POINT(5 5)", "POINT(15 15)"]                   | "POINT(5 5)"                   |  ["POINT(55 55)", "POINT(65 65)"]
+5    | 0.0                | 0.0               | 6210584.082742179   | 6210584.082742179 | ["POINT(0 0)","POINT(11 11)"]                    | "POINT(0 0)"                   |  ["POINT(55 55)", "POINT(65 65)"]
+6    | 1412359.654429245  | 0.0               | 6511042.96320646    | 6511042.96320646  | ["POINT(-9 -19)","POINT(9 9)"]                   | "POINT(-9 -19)"                |  ["POINT(55 55)", "POINT(65 65)"]
+7    | 1756128.8127267    | 0.0               | 6157935.487780502   | 6157935.487780502 | ["POINT(5 15)", "POINT(15 5)"]                   | "POINT(5 15)"                  |  ["POINT(55 55)", "POINT(65 65)"]
+8    | 1223145.8694147274 | 0.0               | 6823348.604540896   | 6823348.604540896 | ["POINT(0 11)","POINT(11 0)"]                    | "POINT(0 11)"                  |  ["POINT(55 55)", "POINT(65 65)"]
+9    | 2329806.5462053656 | 0.0               | 6014935.534393433   | 6014935.534393433 | "POINT(19 9)"                                    | "POINT(19 9)"                  |  ["POINT(55 55)", "POINT(65 65)"]
+10   | 2329806.5462053656 | 0.0               | 6014935.534393433   | 6014935.534393433 | ["POINT(-9 -19)","POINT(19 9)"]                  | "POINT(-9 -19)"                |  ["POINT(55 55)", "POINT(65 65)"]
+11   | 0.0                | 0.0               | 0.0                 | 0.0               | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] | ["POINT(0 0)", "POINT(55 55)"] |  ["POINT(55 55)", "POINT(65 65)"]
+12   | 0.0                | 0.0               | 0.0                 | 0.0               | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] | ["POINT(0 0)", "POINT(55 54)"] |  ["POINT(55 55)", "POINT(65 65)"]
+;
+
+whereSpatialDistanceTwoFields
+required_capability: st_distance
+required_capability: spatial_distance_supports_multivalues
+
+FROM multivalue_points
+| WHERE ST_DISTANCE(location, disjoint) < 6200000
+| KEEP id, location
+| SORT id
+;
+
+id:l | location:geo_point
+4    | ["POINT(5 5)", "POINT(15 15)"]
+7    | ["POINT(5 15)", "POINT(15 5)"]
+9    | "POINT(19 9)"
+10   | ["POINT(-9 -19)","POINT(19 9)"]
+11   | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"]
+12   | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"]
+;
+
+whereSpatialDistanceKeyword
+required_capability: st_distance
+required_capability: spatial_distance_supports_multivalues
+
+FROM multivalue_points
+| EVAL lk=lk::geo_point
+| WHERE ST_DISTANCE(lk, disjoint) < 6200000
+| KEEP id, location
+| SORT id
+;
+
+id:l | location:geo_point
+4    | ["POINT(5 5)", "POINT(15 15)"]
+7    | ["POINT(5 15)", "POINT(15 5)"]
+9    | "POINT(19 9)"
+10   | ["POINT(-9 -19)","POINT(19 9)"]
+11   | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"]
+12   | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"]
+;
+
+spatialDistanceFilterTwoFieldStats
+required_capability: st_distance
+required_capability: spatial_distance_supports_multivalues
+
+FROM multivalue_points
+| EVAL distance_disjoint=ST_DISTANCE(location, disjoint)
+| WHERE distance_disjoint > 0
+| EVAL kkm = TO_INTEGER(distance_disjoint / 1000000)
+| STATS c=COUNT(*) BY kkm
+| SORT kkm ASC
+;
+
+c:long | kkm:integer
+5      | 6
+5      | 7
+1      | 8
+;
+
 whereIntersectsPolygon
 required_capability: st_intersects
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec
index 2f6313925032e..6dc03d0debcfa 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec
@@ -49,20 +49,6 @@ book_no:keyword | title:text
 7350            | Return of the Shadow
 ;
 
-qstrWithDisjunction
-required_capability: qstr_function
-
-from books 
-| where qstr("title:Return") or year > 2020
-| keep book_no, title;
-ignoreOrder:true
-
-book_no:keyword | title:text
-2714            | Return of the King Being the Third Part of The Lord of the Rings
-6818            | Hadji Murad                                                     
-7350            | Return of the Shadow         
-;
-
 qstrWithConjunction
 required_capability: qstr_function
 
@@ -88,17 +74,16 @@ card:keyword   |host:keyword   |ip0:ip                   |ip1:ip
 eth1           |beta           |127.0.0.1                |127.0.0.2
 ;
 
-qstrWithFunctionNotPushedToLucene
+qstrWithNonPushableConjunction
 required_capability: qstr_function
 
 from books 
-| where qstr("title: rings") and length(description) > 600 
+| where qstr("title: Rings") and length(title) > 75
 | keep book_no, title;
 ignoreOrder:true
 
 book_no:keyword | title:text
-2675            | The Lord of the Rings - Boxed Set                               
-2714            | Return of the King Being the Third Part of The Lord of the Rings     
+4023            |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings
 ;
 
 qstrWithMultipleWhereClauses
@@ -114,3 +99,55 @@ book_no:keyword | title:text
 4023            | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings           
 7140            | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1)
 ;
+
+
+matchMultivaluedTextField
+required_capability: match_function
+
+from employees 
+| where qstr("job_positions: (Tech Lead) AND job_positions:(Reporting Analyst)") 
+| keep emp_no, first_name, last_name;
+ignoreOrder:true
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10004          | Chirstian          | Koblick        
+10010          | Duangkaew          | Piveteau       
+10011          | Mary               | Sluis          
+10088          | Jungsoon           | Syrzycki       
+10093          | Sailaja            | Desikan        
+10097          | Remzi              | Waschkowski    
+;
+
+matchMultivaluedNumericField
+required_capability: match_function
+
+from employees 
+| where qstr("salary_change: [14 TO *]") 
+| keep emp_no, first_name, last_name, salary_change;
+ignoreOrder:true
+
+emp_no:integer | first_name:keyword | last_name:keyword | salary_change:double
+10003          | Parto              | Bamford           | [12.82, 14.68]              
+10015          | Guoxiang           | Nooteboom         | [12.4, 14.25]               
+10023          | Bojan              | Montemayor        | [0.8, 14.63]                
+10040          | Weiyi              | Meriste           | [-8.94, 1.92, 6.97, 14.74]  
+10061          | Tse                | Herber            | [-2.58, -0.95, 14.39]       
+10065          | Satosi             | Awdeh             | [-9.81, -1.47, 14.44]       
+10099          | Valter             | Sullins           | [-8.78, -3.98, 10.71, 14.26]
+;
+
+testMultiValuedFieldWithConjunction
+required_capability: match_function
+
+from employees 
+| where (qstr("job_positions: (Data Scientist) OR job_positions:(Support Engineer)")) and gender == "F"
+| keep emp_no, first_name, last_name;
+ignoreOrder:true
+
+emp_no:integer | first_name:keyword | last_name:keyword  
+10023          | Bojan              | Montemayor
+10041          | Uri                | Lenart
+10044          | Mingsen            | Casley
+10053          | Sanjiv             | Zschoche
+10069          | Margareta          | Bierman
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv
new file mode 100644
index 0000000000000..c6de9a208e9a7
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv
@@ -0,0 +1,4 @@
+_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long
+1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001
+2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002
+3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec
new file mode 100644
index 0000000000000..683bcdc3f7490
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec
@@ -0,0 +1,175 @@
+simple
+required_capability: semantic_text_type
+
+FROM semantic_text
+| KEEP semantic_text_field
+| sort semantic_text_field asc;
+
+semantic_text_field:semantic_text
+all we have to decide is what to do with the time that is given to us
+be excellent to each other
+live long and prosper
+;
+
+simpleWithUnicode
+required_capability: semantic_text_type
+
+FROM semantic_text
+| KEEP st_unicode
+| SORT st_unicode
+;
+
+st_unicode:semantic_text
+你吃饭了吗
+["谢谢", "对不起我的中文不好"]
+null
+;
+
+mvExpand
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| KEEP _id, st_multi_value
+| MV_EXPAND st_multi_value
+| SORT st_multi_value
+;
+
+_id:keyword | st_multi_value:semantic_text
+1           | Hello there!
+1           | This is a random value
+2           | bye bye!
+1           | for testing purposes
+2           | nice to meet you
+3           | null
+;
+
+withDropAndKeep
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| KEEP _id, semantic_text_field, st_double
+| DROP st_double
+| SORT _id
+;
+
+_id:keyword | semantic_text_field:semantic_text
+1           | live long and prosper
+2           | all we have to decide is what to do with the time that is given to us
+3           | be excellent to each other
+;
+
+rename
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| RENAME semantic_text_field AS my_field
+| KEEP _id, my_field
+| SORT _id
+;
+
+_id:keyword | my_field:semantic_text
+1           | live long and prosper
+2           | all we have to decide is what to do with the time that is given to us
+3           | be excellent to each other
+;
+
+eval
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| EVAL my_field = semantic_text_field
+| KEEP _id, my_field
+| SORT _id
+;
+
+_id:keyword | my_field:semantic_text
+1           | live long and prosper
+2           | all we have to decide is what to do with the time that is given to us
+3           | be excellent to each other
+;
+
+simpleStats
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| STATS COUNT(*)
+;
+
+COUNT(*):long
+3
+;
+
+statsWithGrouping
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| STATS COUNT(*) BY st_version
+| SORT st_version
+;
+
+COUNT(*):long | st_version:semantic_text
+1             | 1.2.3
+1             | 9.0.0
+1             | null
+;
+
+withDropKeepStatsMvExpandRenameSortLimit
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| KEEP _id, semantic_text_field, st_multi_value
+| DROP semantic_text_field
+| RENAME st_multi_value AS my_field
+| MV_EXPAND my_field
+| STATS COUNT(*) BY my_field
+| SORT my_field
+| LIMIT 3
+;
+
+COUNT(*):long | my_field:semantic_text
+1             | Hello there!
+1             | This is a random value
+1             | bye bye!
+;
+
+simpleWithLongValue
+required_capability: semantic_text_type
+
+FROM semantic_text
+| KEEP value, semantic_text_field
+| SORT value
+;
+
+value:long | semantic_text_field:semantic_text
+1001            | live long and prosper
+1002            | all we have to decide is what to do with the time that is given to us
+1003            | be excellent to each other
+;
+
+simpleWithText
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| KEEP description, semantic_text_field
+| SORT description
+;
+
+description:text     | semantic_text_field:semantic_text
+"some description1"  | live long and prosper
+"some description2"  | all we have to decide is what to do with the time that is given to us
+"some description3"  | be excellent to each other
+;
+
+simpleWithKeyword
+required_capability: semantic_text_type
+
+FROM semantic_text METADATA _id
+| KEEP host, semantic_text_field
+| SORT host
+;
+
+host:keyword | semantic_text_field:semantic_text
+"host1"      | live long and prosper
+"host2"      | all we have to decide is what to do with the time that is given to us
+"host3"      | be excellent to each other
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
index 4c40808a4ff96..c1c4538c7393d 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
@@ -616,6 +616,42 @@ location:geo_point  | city_location:geo_point  |  count:long
 POINT (0 0)         | POINT (0 0)              |  1
 ;
 
+airportCityLocationPointIntersectionCentroidGroups
+required_capability: st_intersects
+
+FROM airports_mp
+| WHERE ST_INTERSECTS(location, city_location)
+| STATS location=ST_CENTROID_AGG(location), city_location=ST_CENTROID_AGG(city_location), count=COUNT() BY country
+;
+
+location:geo_point  | city_location:geo_point  |  count:long | country:k
+POINT (0 0)         | POINT (0 0)              |  1          | Atlantis
+;
+
+airportCityLocationPointIntersectionNullCentroid
+required_capability: st_intersects
+required_capability: spatial_centroid_no_records
+
+FROM airports
+| WHERE ST_INTERSECTS(location, city_location)
+| STATS location=ST_CENTROID_AGG(location), city_location=ST_CENTROID_AGG(city_location), count=COUNT()
+;
+
+location:geo_point  | city_location:geo_point  |  count:long
+null                | null                     |  0
+;
+
+airportCityLocationPointIntersectionNullCentroidGroups
+required_capability: st_intersects
+
+FROM airports
+| WHERE ST_INTERSECTS(location, city_location)
+| STATS location=ST_CENTROID_AGG(location), city_location=ST_CENTROID_AGG(city_location), count=COUNT() BY country
+;
+
+location:geo_point  | city_location:geo_point  |  count:long | country:k
+;
+
 ###############################################
 # Tests for ST_DISJOINT on GEO_POINT type
 
@@ -1203,7 +1239,7 @@ count:long | country:k
 1          | Poland
 ;
 
-airportsSortCityName
+airportsSortCityName#[skip:-8.13.3, reason:fixed in 8.13]
 FROM airports
 | SORT abbrev
 | LIMIT 5
@@ -1948,14 +1984,15 @@ wkt:keyword   | pt:cartesian_point
 
 cartesianCentroidFromAirportsAfterPointContainsPolygonPredicate
 required_capability: st_contains_within
+required_capability: spatial_centroid_no_records
 
 FROM airports_web
 | WHERE ST_CONTAINS(location, TO_CARTESIANSHAPE("POLYGON((4700000 1600000, 4800000 1600000, 4800000 1700000, 4700000 1700000, 4700000 1600000))"))
 | STATS centroid=ST_CENTROID_AGG(location), count=COUNT()
 ;
 
-centroid:cartesian_point     | count:long
-POINT (NaN NaN)              | 0
+centroid:cartesian_point | count:long
+null                     | 0
 ;
 
 cartesianPointContainsPolygonPredicate
@@ -2222,17 +2259,19 @@ wkt:keyword    | pt:cartesian_point  | distance:double
 
 twoCitiesPointDistanceCartesian
 required_capability: st_distance
+required_capability: spatial_distance_supports_multivalues
 
 ROW p1 = TO_CARTESIANPOINT("POINT(-90.82814 29.79511)"), p2 = TO_CARTESIANPOINT("POINT(-90.79731509999999 29.8835389)")
 | EVAL d = ST_DISTANCE(p1, p2)
 ;
 
 p1:cartesian_point          | p2:cartesian_point                     | d:double
-POINT (-90.82814 29.79511)  | POINT (-90.79731509999999 29.8835389)  | 0.09364744959271905
+POINT (-90.82814 29.79511)  | POINT (-90.79731509999999 29.8835389)  | 0.09364636296011444
 ;
 
 airportCartesianCityLocationPointDistance
 required_capability: st_distance
+required_capability: spatial_distance_supports_multivalues
 
 FROM airports_web
 | EVAL distance = ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)"))
@@ -2240,12 +2279,13 @@ FROM airports_web
 | STATS distance=AVG(distance), min=min(distance), max=max(distance), count=COUNT()
 ;
 
-distance:double    | min:double       | max:double        | count:long
-676858.3463435044  | 7358.02077507206 | 971112.9731194031 | 12
+distance:double    | min:double        | max:double        | count:long
+676858.3629952326  | 7358.012830411482 | 971113.1663946278 | 12
 ;
 
 airportCartesianDistanceToCityCopenhagen
 required_capability: st_distance
+required_capability: spatial_distance_supports_multivalues
 
 // tag::st_distance-airports_web[]
 FROM airports_web
@@ -2257,7 +2297,7 @@ FROM airports_web
 
 // tag::st_distance-airports_web-result[]
 abbrev:k | name:text   | location:cartesian_point                   | distance:d
-CPH      | Copenhagen  | POINT(1408119.2975413958 7484813.53657096) | 7358.02077507206
+CPH      | Copenhagen  | POINT(1408119.2975413958 7484813.53657096) | 7358.012830411482
 // end::st_distance-airports_web-result[]
 ;
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
index 8a2e9b402fbca..2dc21a86e6394 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
@@ -2290,3 +2290,368 @@ from employees
 m:integer |a:double |x:integer
 74999     |48249.0  |0
 ;
+
+statsWithFilterOnGroups
+required_capability: fix_filter_pushdown_past_stats
+FROM employees
+| STATS v = VALUES(emp_no) by job_positions | WHERE job_positions == "Accountant" | MV_EXPAND v | SORT v
+;
+
+v:integer | job_positions:keyword
+    10001 | Accountant
+    10012 | Accountant
+    10016 | Accountant
+    10023 | Accountant
+    10025 | Accountant
+    10028 | Accountant
+    10034 | Accountant
+    10037 | Accountant
+    10044 | Accountant
+    10045 | Accountant
+    10050 | Accountant
+    10051 | Accountant
+    10066 | Accountant
+    10081 | Accountant
+    10085 | Accountant
+    10089 | Accountant
+    10092 | Accountant
+    10094 | Accountant
+;
+
+
+statsWithFiltering
+required_capability: per_agg_filtering
+from employees
+| stats max = max(salary), max_f = max(salary) where salary < 50000, max_a = max(salary) where salary > 100,
+        min = min(salary), min_f = min(salary) where salary > 50000, min_a = min(salary) where salary > 100  
+;
+
+max:integer |max_f:integer |max_a:integer | min:integer | min_f:integer | min_a:integer
+74999       |49818         |74999         | 25324       | 50064         | 25324
+;
+
+statsWithEverythingFiltered
+required_capability: per_agg_filtering
+from employees
+| stats max = max(salary), max_a = max(salary) where salary < 100,
+        min = min(salary), min_a = min(salary) where salary > 99999  
+;
+
+max:integer |max_a:integer|min:integer | min_a:integer
+74999       |null         |25324       | null        
+;
+
+statsWithNullFilter
+required_capability: per_agg_filtering
+from employees
+| stats max = max(salary), max_a = max(salary) where null,
+        min = min(salary), min_a = min(salary) where to_string(null) == "abc"  
+;
+
+max:integer |max_a:integer|min:integer | min_a:integer
+74999       |null         |25324       | null        
+;
+
+statsWithBasicExpressionFiltered
+required_capability: per_agg_filtering
+from employees
+| stats max = max(salary), max_f = max(salary) where salary < 50000,
+        min = min(salary), min_f = min(salary) where salary > 50000,
+        exp_p = max(salary) + 10000 where salary < 50000,
+        exp_m = min(salary) % 10000 where salary > 50000  
+;
+
+max:integer |max_f:integer|min:integer | min_f:integer|exp_p:integer | exp_m:integer
+74999       |49818        |25324       | 50064        |59818         | 64
+;
+
+statsWithExpressionOverFilters
+required_capability: per_agg_filtering
+from employees
+| stats max = max(salary), max_f = max(salary) where salary < 50000,
+        min = min(salary), min_f = min(salary) where salary > 50000,
+        exp_gt = max(salary) - min(salary) where salary > 50000,
+        exp_lt = max(salary) - min(salary) where salary < 50000
+          
+;
+
+max:integer |max_f:integer | min:integer | min_f:integer |exp_gt:integer | exp_lt:integer
+74999       |49818         | 25324       | 50064         |24935          | 24494        
+;
+
+
+statsWithExpressionOfExpressionsOverFilters
+required_capability: per_agg_filtering
+from employees
+| stats max = max(salary + 1), max_f = max(salary + 2) where salary < 50000,
+        min = min(salary - 1), min_f = min(salary - 2) where salary > 50000,
+        exp_gt = max(salary + 3) - min(salary - 3) where salary > 50000,
+        exp_lt = max(salary + 4) - min(salary - 4) where salary < 50000
+          
+;
+
+max:integer |max_f:integer | min:integer | min_f:integer |exp_gt:integer | exp_lt:integer
+75000       |49820         | 25323       | 50062         |24941          | 24502        
+;
+
+statsWithSubstitutedExpressionOverFilters
+required_capability: per_agg_filtering
+from employees
+| stats sum = sum(salary), s_l = sum(salary) where salary < 50000, s_u = sum(salary) where salary > 50000,
+        count = count(salary), c_l = count(salary) where salary < 50000, c_u = count(salary) where salary > 50000, 
+        avg = round(avg(salary), 2), a_l = round(avg(salary), 2) where salary < 50000, a_u = round(avg(salary),2) where salary > 50000
+;
+
+sum:l   |s_l:l   | s_u:l   | count:l |c_l:l |c_u:l |avg:double |a_l:double | a_u:double
+4824855 |2220951 | 2603904 | 100     |58    |42    |48248.55   |38292.26   | 61997.71
+;
+
+
+statsWithFilterAndGroupBy
+required_capability: per_agg_filtering
+from employees
+| stats m = max(height),
+        m_f = max(height + 1) where gender == "M" OR is_rehired is null
+        BY gender, is_rehired
+| sort gender, is_rehired
+;
+
+m:d |m_f:d |gender:s|is_rehired:bool
+2.1 |null  |F       |false    
+2.1 |null  |F       |true     
+1.85|2.85  |F       |null     
+2.1 |3.1   |M       |false    
+2.1 |3.1   |M       |true     
+2.01|3.01  |M       |null     
+2.06|null  |null    |false    
+1.97|null  |null    |true     
+1.99|2.99  |null    |null     
+;
+
+statsWithFilterOnGroupBy
+required_capability: per_agg_filtering
+from employees
+| stats m_f = max(height) where gender == "M" BY gender
+| sort gender
+;
+
+m_f:d   |gender:s
+null    |F
+2.1     |M
+null    |null
+;
+
+statsWithGroupByLiteral
+required_capability: per_agg_filtering
+from employees
+| stats m = max(languages) by salary = 2
+;
+
+m:i |salary:i
+5   |2
+;
+
+
+statsWithFilterOnSameColumn
+required_capability: per_agg_filtering
+from employees
+| stats m = max(languages), m_f = max(languages) where salary > 50000 by salary = 2
+| sort salary
+;
+
+m:i |m_f:i |salary:i
+5   |null  |2
+;
+
+# the query is reused below in a multi-stats
+statsWithFilteringAndGrouping
+required_capability: per_agg_filtering
+from employees
+| stats c = count(), c_f = count(languages) where l > 1, 
+        m_f = max(height) where salary > 50000 
+        by l = languages
+| sort c
+;
+
+c:l |c_f:l |m_f:d |l:i
+10  |0     |2.08  |null        
+15  |0     |2.06  |1           
+17  |17    |2.1   |3           
+18  |18    |1.83  |4           
+19  |19    |2.03  |2           
+21  |21    |2.1   |5           
+;
+ 
+multiStatsWithFiltering
+required_capability: per_agg_filtering
+from employees
+| stats c = count(), c_f = count(languages) where l > 1, 
+        m_f = max(height) where salary > 50000 
+        by l = languages
+| stats c2 = count(), c2_f = count() where m_f > 2.06 , m2 = max(l), m2_f = max(l) where l > 1 by c
+| sort c
+;
+
+c2:l |c2_f:l |m2:i |m2_f:i |c:l
+1    |1      |null |null   |10
+1    |0      |1    |null   |15
+1    |1      |3    |3      |17
+1    |0      |4    |4      |18
+1    |0      |2    |2      |19
+1    |1      |5    |5      |21
+;
+
+
+statsByConstantExpressionNoAggs
+required_capability: fix_stats_by_foldable_expression
+FROM employees | eval x = [1,2,3], y = 5 + 6 | stats by y+1
+;
+
+y+1:integer
+12
+;
+
+
+statsByConstantExpressionNoAggsWithAlias
+required_capability: fix_stats_by_foldable_expression
+FROM employees | eval x = [1,2,3], y = 5 + 6 | stats by yy = y+1
+;
+
+yy:integer
+12
+;
+
+statsByConstantExpression
+required_capability: fix_stats_by_foldable_expression
+FROM employees | eval x = [1,2,3], y = 5 + 6 | stats m = max(y) by y+1
+;
+
+m:integer | y+1:integer
+11        | 12
+;
+
+filterIsAlwaysTrue
+required_capability: per_agg_filtering
+FROM employees
+| STATS max = max(salary) WHERE salary > 0
+;
+
+max:integer
+74999
+;
+
+filterIsAlwaysFalse
+required_capability: per_agg_filtering
+FROM employees
+| STATS max = max(salary) WHERE first_name == ""
+;
+
+max:integer
+null
+;
+
+filterSometimesMatches
+required_capability: per_agg_filtering
+FROM employees
+| STATS max = max(salary) WHERE first_name IS NULL
+;
+
+max:integer
+70011
+;
+
+groupingFilterIsAlwaysTrue
+required_capability: per_agg_filtering
+FROM employees
+| MV_EXPAND job_positions
+| STATS max = max(salary) WHERE salary > 0 BY job_positions = SUBSTRING(job_positions, 1, 1)
+| SORT job_positions
+| LIMIT 4
+;
+
+max:integer | job_positions:keyword
+74970       | A
+58121       | B
+74999       | D
+58715       | H
+;
+
+groupingFilterIsAlwaysFalse
+required_capability: per_agg_filtering
+FROM employees
+| MV_EXPAND job_positions
+| STATS max = max(salary) WHERE first_name == "" BY job_positions = SUBSTRING(job_positions, 1, 1)
+| SORT job_positions
+| LIMIT 4
+;
+
+max:integer | job_positions:keyword
+null        | A
+null        | B
+null        | D
+null        | H
+;
+
+groupingFilterSometimesMatches
+required_capability: per_agg_filtering
+FROM employees
+| MV_EXPAND job_positions
+| STATS max = max(salary) WHERE first_name IS NULL BY job_positions = SUBSTRING(job_positions, 1, 1)
+| SORT job_positions
+| LIMIT 4
+;
+
+max:integer | job_positions:keyword
+62233       | A
+39878       | B
+67492       | D
+null        | H
+;
+
+groupingByOrdinalsFilterIsAlwaysTrue
+required_capability: per_agg_filtering
+required_capability: per_agg_filtering_ords
+FROM employees
+| STATS max = max(salary) WHERE salary > 0 BY job_positions
+| SORT job_positions
+| LIMIT 4
+;
+
+max:integer | job_positions:keyword
+74970       | Accountant
+69904       | Architect
+58121       | Business Analyst
+74999       | Data Scientist
+;
+
+groupingByOrdinalsFilterIsAlwaysFalse
+required_capability: per_agg_filtering
+required_capability: per_agg_filtering_ords
+FROM employees
+| STATS max = max(salary) WHERE first_name == "" BY job_positions
+| SORT job_positions
+| LIMIT 4
+;
+
+max:integer | job_positions:keyword
+null        | Accountant
+null        | Architect
+null        | Business Analyst
+null        | Data Scientist
+;
+
+groupingByOrdinalsFilterSometimesMatches
+required_capability: per_agg_filtering
+required_capability: per_agg_filtering_ords
+FROM employees
+| STATS max = max(salary) WHERE first_name IS NULL BY job_positions
+| SORT job_positions
+| LIMIT 4
+;
+
+max:integer | job_positions:keyword
+39878       | Accountant
+62233       | Architect
+39878       | Business Analyst
+67492       | Data Scientist
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
index 5313e6630c75d..00fa2fddb2106 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
@@ -1236,6 +1236,7 @@ off_on_holiday:keyword | back_home_again:keyword
 
 reverseGraphemeClusters
 required_capability: fn_reverse
+required_capability: fn_reverse_grapheme_clusters
 ROW message = "áéíóúàèìòùâêîôû😊👍🏽🎉💖कंठाी" | EVAL message_reversed = REVERSE(message);
 
 message:keyword                | message_reversed:keyword
@@ -1799,3 +1800,59 @@ warning:Line 1:29: java.lang.IllegalArgumentException: single-value function enc
 x:keyword
 null
 ;
+
+
+likeEscapingSingleQuotes
+// tag::likeEscapingSingleQuotes[]
+ROW message = "foo * bar"
+| WHERE message LIKE "foo \\* bar"
+// end::likeEscapingSingleQuotes[]
+;
+
+// tag::likeEscapingSingleQuotes-result[]
+message:keyword
+foo * bar
+// end::likeEscapingSingleQuotes-result[]
+;
+
+
+likeEscapingTripleQuotes
+// tag::likeEscapingTripleQuotes[]
+ROW message = "foo * bar"
+| WHERE message LIKE """foo \* bar"""
+// end::likeEscapingTripleQuotes[]
+;
+
+// tag::likeEscapingTripleQuotes-result[]
+message:keyword
+foo * bar
+// end::likeEscapingTripleQuotes-result[]
+;
+
+
+rlikeEscapingSingleQuotes
+// tag::rlikeEscapingSingleQuotes[]
+ROW message = "foo ( bar"
+| WHERE message RLIKE "foo \\( bar"
+// end::rlikeEscapingSingleQuotes[]
+;
+
+// tag::rlikeEscapingSingleQuotes-result[]
+message:keyword
+foo ( bar
+// end::rlikeEscapingSingleQuotes-result[]
+;
+
+
+rlikeEscapingTripleQuotes
+// tag::rlikeEscapingTripleQuotes[]
+ROW message = "foo ( bar"
+| WHERE message RLIKE """foo \( bar"""
+// end::rlikeEscapingTripleQuotes[]
+;
+
+// tag::rlikeEscapingTripleQuotes-result[]
+message:keyword
+foo ( bar
+// end::rlikeEscapingTripleQuotes-result[]
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec
index 3d4d890546050..e7bf953f5e08d 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec
@@ -135,7 +135,7 @@ null              |Swan             |-8.46                       |-8.46
 Sanjiv            |Zschoche         |[-7.67, -3.25]              |[-3.25, -7.67]               |[-3, -8]             |10053
 ;
 
-sortingOnSwappedFields
+sortingOnSwappedFields#[skip:-8.13.3, reason:fixed in 8.13]
 FROM employees
 | EVAL name = last_name, last_name = first_name, first_name = name
 | WHERE first_name > "B" AND last_name IS NOT NULL
@@ -157,7 +157,7 @@ Brattka      | Charlene          | Brattka
 Bridgland    | Patricio          | Bridgland
 ;
 
-sortingOnSwappedFieldsNoKeep
+sortingOnSwappedFieldsNoKeep#[skip:-8.13.3, reason:fixed in 8.13]
 // Note that this test requires all fields to be returned in order to test a specific code path in physical planning
 FROM employees
 | EVAL name = first_name, first_name = last_name, last_name = name
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json
index dd4073d5dc7cf..39b1b10edd916 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json
@@ -27,6 +27,10 @@
         "message_in": {
           "type": "float",
           "time_series_metric": "counter"
+        },
+        "message_out": {
+          "type": "integer",
+          "time_series_metric": "counter"
         }
       }
     }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
index 452d56680e8da..e8e9f45694e9c 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
@@ -46,6 +47,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -220,7 +222,7 @@ static String enrichVendors(Enrich.Mode mode) {
     public void testWithHostsPolicy() {
         for (var mode : Enrich.Mode.values()) {
             String query = "FROM events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os";
-            try (EsqlQueryResponse resp = runQuery(query)) {
+            try (EsqlQueryResponse resp = runQuery(query, null)) {
                 List<List<Object>> rows = getValuesList(resp);
                 assertThat(
                     rows,
@@ -237,9 +239,14 @@ public void testWithHostsPolicy() {
                 assertFalse(resp.getExecutionInfo().isCrossClusterSearch());
             }
         }
+
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         for (var mode : Enrich.Mode.values()) {
             String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os";
-            try (EsqlQueryResponse resp = runQuery(query)) {
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
                 List<List<Object>> rows = getValuesList(resp);
                 assertThat(
                     rows,
@@ -255,6 +262,7 @@ public void testWithHostsPolicy() {
                     )
                 );
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
                 assertThat(executionInfo.clusterAliases(), equalTo(Set.of("c1", "c2")));
                 assertCCSExecutionInfoDetails(executionInfo);
             }
@@ -262,7 +270,7 @@ public void testWithHostsPolicy() {
 
         for (var mode : Enrich.Mode.values()) {
             String query = "FROM *:events,events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os";
-            try (EsqlQueryResponse resp = runQuery(query)) {
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
                 List<List<Object>> rows = getValuesList(resp);
                 assertThat(
                     rows,
@@ -278,6 +286,7 @@ public void testWithHostsPolicy() {
                     )
                 );
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
                 assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
                 assertCCSExecutionInfoDetails(executionInfo);
             }
@@ -285,6 +294,10 @@ public void testWithHostsPolicy() {
     }
 
     public void testEnrichHostsAggThenEnrichVendorCoordinator() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         for (var hostMode : Enrich.Mode.values()) {
             String query = String.format(Locale.ROOT, """
                 FROM *:events,events
@@ -295,7 +308,7 @@ public void testEnrichHostsAggThenEnrichVendorCoordinator() {
                 | stats c = SUM(c) by vendor
                 | sort vendor
                 """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR));
-            try (EsqlQueryResponse resp = runQuery(query)) {
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
                 assertThat(
                     getValuesList(resp),
                     equalTo(
@@ -309,6 +322,7 @@ public void testEnrichHostsAggThenEnrichVendorCoordinator() {
                     )
                 );
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
                 assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
                 assertCCSExecutionInfoDetails(executionInfo);
             }
@@ -316,6 +330,10 @@ public void testEnrichHostsAggThenEnrichVendorCoordinator() {
     }
 
     public void testEnrichTwiceThenAggs() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         for (var hostMode : Enrich.Mode.values()) {
             String query = String.format(Locale.ROOT, """
                 FROM *:events,events
@@ -325,7 +343,7 @@ public void testEnrichTwiceThenAggs() {
                 | stats c = COUNT(*) by vendor
                 | sort vendor
                 """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR));
-            try (EsqlQueryResponse resp = runQuery(query)) {
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
                 assertThat(
                     getValuesList(resp),
                     equalTo(
@@ -339,6 +357,7 @@ public void testEnrichTwiceThenAggs() {
                     )
                 );
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
                 assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
                 assertCCSExecutionInfoDetails(executionInfo);
             }
@@ -346,6 +365,10 @@ public void testEnrichTwiceThenAggs() {
     }
 
     public void testEnrichCoordinatorThenAny() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         String query = String.format(Locale.ROOT, """
             FROM *:events,events
             | eval ip= TO_STR(host)
@@ -354,7 +377,7 @@ public void testEnrichCoordinatorThenAny() {
             | stats c = COUNT(*) by vendor
             | sort vendor
             """, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.ANY));
-        try (EsqlQueryResponse resp = runQuery(query)) {
+        try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
             assertThat(
                 getValuesList(resp),
                 equalTo(
@@ -368,12 +391,17 @@ public void testEnrichCoordinatorThenAny() {
                 )
             );
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
             assertCCSExecutionInfoDetails(executionInfo);
         }
     }
 
     public void testEnrichCoordinatorWithVendor() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         for (Enrich.Mode hostMode : Enrich.Mode.values()) {
             String query = String.format(Locale.ROOT, """
                 FROM *:events,events
@@ -383,7 +411,7 @@ public void testEnrichCoordinatorWithVendor() {
                 | stats c = COUNT(*) by vendor
                 | sort vendor
                 """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR));
-            try (EsqlQueryResponse resp = runQuery(query)) {
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
                 assertThat(
                     getValuesList(resp),
                     equalTo(
@@ -397,6 +425,7 @@ public void testEnrichCoordinatorWithVendor() {
                     )
                 );
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
                 assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
                 assertCCSExecutionInfoDetails(executionInfo);
             }
@@ -405,6 +434,10 @@ public void testEnrichCoordinatorWithVendor() {
     }
 
     public void testEnrichRemoteWithVendor() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) {
             var query = String.format(Locale.ROOT, """
                 FROM *:events,events
@@ -414,7 +447,7 @@ public void testEnrichRemoteWithVendor() {
                 | stats c = COUNT(*) by vendor
                 | sort vendor
                 """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE));
-            try (EsqlQueryResponse resp = runQuery(query)) {
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
                 assertThat(
                     getValuesList(resp),
                     equalTo(
@@ -430,6 +463,46 @@ public void testEnrichRemoteWithVendor() {
                     )
                 );
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+                assertCCSExecutionInfoDetails(executionInfo);
+            }
+        }
+    }
+
+    public void testEnrichRemoteWithVendorNoSort() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
+        for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) {
+            var query = String.format(Locale.ROOT, """
+                FROM *:events,events
+                | LIMIT 100
+                | eval ip= TO_STR(host)
+                | %s
+                | %s
+                | stats c = COUNT(*) by vendor
+                """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE));
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                var values = getValuesList(resp);
+                values.sort(Comparator.comparing(o -> (String) o.get(1), Comparator.nullsLast(Comparator.naturalOrder())));
+                assertThat(
+                    values,
+                    equalTo(
+                        List.of(
+                            List.of(6L, "Apple"),
+                            List.of(7L, "Microsoft"),
+                            List.of(1L, "Redhat"),
+                            List.of(2L, "Samsung"),
+                            List.of(1L, "Sony"),
+                            List.of(2L, "Suse"),
+                            Arrays.asList(3L, (String) null)
+                        )
+                    )
+                );
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
                 assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
                 assertCCSExecutionInfoDetails(executionInfo);
             }
@@ -437,26 +510,72 @@ public void testEnrichRemoteWithVendor() {
     }
 
     public void testTopNThenEnrichRemote() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         String query = String.format(Locale.ROOT, """
             FROM *:events,events
             | eval ip= TO_STR(host)
-            | SORT ip
+            | SORT timestamp, user, ip
             | LIMIT 5
-            | %s
+            | %s | KEEP host, timestamp, user, os
             """, enrichHosts(Enrich.Mode.REMOTE));
-        var error = expectThrows(VerificationException.class, () -> runQuery(query).close());
-        assertThat(error.getMessage(), containsString("ENRICH with remote policy can't be executed after LIMIT"));
+        try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+            assertThat(
+                getValuesList(resp),
+                equalTo(
+                    List.of(
+                        List.of("192.168.1.2", 1L, "andres", "Windows"),
+                        List.of("192.168.1.3", 1L, "matthew", "MacOS"),
+                        Arrays.asList("192.168.1.25", 1L, "park", (String) null),
+                        List.of("192.168.1.5", 2L, "akio", "Android"),
+                        List.of("192.168.1.6", 2L, "sergio", "iOS")
+                    )
+                )
+            );
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+            assertCCSExecutionInfoDetails(executionInfo);
+        }
     }
 
     public void testLimitThenEnrichRemote() {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         String query = String.format(Locale.ROOT, """
             FROM *:events,events
-            | LIMIT 10
+            | LIMIT 25
             | eval ip= TO_STR(host)
-            | %s
+            | %s | KEEP host, timestamp, user, os
             """, enrichHosts(Enrich.Mode.REMOTE));
-        var error = expectThrows(VerificationException.class, () -> runQuery(query).close());
-        assertThat(error.getMessage(), containsString("ENRICH with remote policy can't be executed after LIMIT"));
+        try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+            var values = getValuesList(resp);
+            values.sort(
+                Comparator.comparingLong((List<Object> o) -> (Long) o.get(1))
+                    .thenComparing(o -> (String) o.get(0))
+                    .thenComparing(o -> (String) o.get(2))
+            );
+            assertThat(
+                values.subList(0, 5),
+                equalTo(
+                    List.of(
+                        List.of("192.168.1.2", 1L, "andres", "Windows"),
+                        Arrays.asList("192.168.1.25", 1L, "park", (String) null),
+                        List.of("192.168.1.3", 1L, "matthew", "MacOS"),
+                        List.of("192.168.1.5", 2L, "akio", "Android"),
+                        List.of("192.168.1.5", 2L, "simon", "Android")
+                    )
+                )
+            );
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+            assertCCSExecutionInfoDetails(executionInfo);
+        }
     }
 
     public void testAggThenEnrichRemote() {
@@ -468,7 +587,7 @@ public void testAggThenEnrichRemote() {
             | %s
             | sort vendor
             """, enrichHosts(Enrich.Mode.ANY), enrichVendors(Enrich.Mode.REMOTE));
-        var error = expectThrows(VerificationException.class, () -> runQuery(query).close());
+        var error = expectThrows(VerificationException.class, () -> runQuery(query, randomBoolean()).close());
         assertThat(error.getMessage(), containsString("ENRICH with remote policy can't be executed after STATS"));
     }
 
@@ -480,20 +599,23 @@ public void testEnrichCoordinatorThenEnrichRemote() {
             | %s
             | sort vendor
             """, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.REMOTE));
-        var error = expectThrows(VerificationException.class, () -> runQuery(query).close());
+        var error = expectThrows(VerificationException.class, () -> runQuery(query, randomBoolean()).close());
         assertThat(
             error.getMessage(),
             containsString("ENRICH with remote policy can't be executed after another ENRICH with coordinator policy")
         );
     }
 
-    protected EsqlQueryResponse runQuery(String query) {
+    protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) {
         EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
         request.query(query);
         request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
         if (randomBoolean()) {
             request.profile(true);
         }
+        if (ccsMetadataInResponse != null) {
+            request.includeCCSMetadata(ccsMetadataInResponse);
+        }
         return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
     }
 
@@ -516,6 +638,15 @@ private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInf
         }
     }
 
+    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
+        return switch (randomIntBetween(1, 3)) {
+            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
+            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
+            case 3 -> new Tuple<>(null, Boolean.FALSE);
+            default -> throw new AssertionError("should not get here");
+        };
+    }
+
     public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin {
 
         public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception {
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
index 4f4f3d112247e..ddd5cff014ed2 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Priority;
@@ -21,13 +20,16 @@
 import org.elasticsearch.compute.operator.DriverProfile;
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.AbstractMultiClustersTestCase;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -79,12 +81,15 @@ public List<Setting<?>> getSettings() {
         }
     }
 
-    public void testSimple() {
+    public void testSuccessfulPathways() {
         Map<String, Object> testClusterInfo = setupTwoClusters();
         int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
         int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
 
-        try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats sum (v)")) {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+        try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats sum (v)", requestIncludeMeta)) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(1));
             assertThat(values.get(0), equalTo(List.of(330L)));
@@ -92,7 +97,9 @@ public void testSimple() {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
-            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+            long overallTookMillis = executionInfo.overallTook().millis();
+            assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
@@ -100,6 +107,7 @@ public void testSimple() {
             assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*"));
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
             assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
             assertThat(remoteCluster.getSkippedShards(), equalTo(0));
@@ -109,13 +117,17 @@ public void testSimple() {
             assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
             assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
             assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
             assertThat(localCluster.getSkippedShards(), equalTo(0));
             assertThat(localCluster.getFailedShards(), equalTo(0));
+
+            // ensure that the _clusters metadata is present only if requested
+            assertClusterMetadataInResponse(resp, responseExpectMeta);
         }
 
-        try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats count(*) by tag | sort tag | keep tag")) {
+        try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats count(*) by tag | sort tag | keep tag", requestIncludeMeta)) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(2));
             assertThat(values.get(0), equalTo(List.of("local")));
@@ -124,7 +136,9 @@ public void testSimple() {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
-            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+            long overallTookMillis = executionInfo.overallTook().millis();
+            assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
@@ -132,6 +146,7 @@ public void testSimple() {
             assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*"));
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
             assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
             assertThat(remoteCluster.getSkippedShards(), equalTo(0));
@@ -141,10 +156,14 @@ public void testSimple() {
             assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
             assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
             assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
             assertThat(localCluster.getSkippedShards(), equalTo(0));
             assertThat(localCluster.getFailedShards(), equalTo(0));
+
+            // ensure that the _clusters metadata is present only if requested
+            assertClusterMetadataInResponse(resp, responseExpectMeta);
         }
     }
 
@@ -153,9 +172,13 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
         int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
         int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
 
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         // since a valid local index was specified, the invalid index on cluster-a does not throw an exception,
         // but instead is simply ignored - ensure this is captured in the EsqlExecutionInfo
-        try (EsqlQueryResponse resp = runQuery("from logs-*,cluster-a:no_such_index | stats sum (v)")) {
+        try (EsqlQueryResponse resp = runQuery("from logs-*,cluster-a:no_such_index | stats sum (v)", requestIncludeMeta)) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(1));
@@ -163,7 +186,9 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
 
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
-            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+            long overallTookMillis = executionInfo.overallTook().millis();
+            assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
@@ -171,6 +196,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index"));
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(0));  // 0 since no matching index, thus no shards to search
             assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
             assertThat(remoteCluster.getSkippedShards(), equalTo(0));
@@ -180,6 +206,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
             assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
             assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
             assertThat(localCluster.getSkippedShards(), equalTo(0));
@@ -188,7 +215,12 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
 
         // since the remote cluster has a valid index expression, the missing local index is ignored
         // make this is captured in the EsqlExecutionInfo
-        try (EsqlQueryResponse resp = runQuery("from no_such_index,*:logs-* | stats count(*) by tag | sort tag | keep tag")) {
+        try (
+            EsqlQueryResponse resp = runQuery(
+                "from no_such_index,*:logs-* | stats count(*) by tag | sort tag | keep tag",
+                requestIncludeMeta
+            )
+        ) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(1));
             assertThat(values.get(0), equalTo(List.of("remote")));
@@ -196,7 +228,9 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
-            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+            long overallTookMillis = executionInfo.overallTook().millis();
+            assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
@@ -204,6 +238,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*"));
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
             assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
             assertThat(remoteCluster.getSkippedShards(), equalTo(0));
@@ -211,8 +246,9 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
             assertThat(localCluster.getIndexExpression(), equalTo("no_such_index"));
-            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(localCluster.getTotalShards(), equalTo(0));
             assertThat(localCluster.getSuccessfulShards(), equalTo(0));
             assertThat(localCluster.getSkippedShards(), equalTo(0));
@@ -223,7 +259,8 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
         // in the index expression of the EsqlExecutionInfo and with an indication that zero shards were searched
         try (
             EsqlQueryResponse resp = runQuery(
-                "FROM no_such_index*,*:no_such_index1,*:no_such_index2,logs-1 | STATS COUNT(*) by tag | SORT tag | KEEP tag"
+                "FROM no_such_index*,*:no_such_index1,*:no_such_index2,logs-1 | STATS COUNT(*) by tag | SORT tag | KEEP tag",
+                requestIncludeMeta
             )
         ) {
             List<List<Object>> values = getValuesList(resp);
@@ -233,7 +270,9 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
-            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+            long overallTookMillis = executionInfo.overallTook().millis();
+            assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
@@ -241,6 +280,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index1,no_such_index2"));
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(0));
             assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
             assertThat(remoteCluster.getSkippedShards(), equalTo(0));
@@ -250,6 +290,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             assertThat(localCluster.getIndexExpression(), equalTo("no_such_index*,logs-1"));
             assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
             assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
             assertThat(localCluster.getSkippedShards(), equalTo(0));
@@ -257,7 +298,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
         }
 
         // wildcard on remote cluster that matches nothing - should be present in EsqlExecutionInfo marked as SKIPPED, no shards searched
-        try (EsqlQueryResponse resp = runQuery("from cluster-a:no_such_index*,logs-* | stats sum (v)")) {
+        try (EsqlQueryResponse resp = runQuery("from cluster-a:no_such_index*,logs-* | stats sum (v)", requestIncludeMeta)) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(1));
@@ -265,7 +306,9 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
 
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
-            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+            long overallTookMillis = executionInfo.overallTook().millis();
+            assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
@@ -273,6 +316,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index*"));
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(0));
             assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
             assertThat(remoteCluster.getSkippedShards(), equalTo(0));
@@ -282,6 +326,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
             assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
             assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
             assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
             assertThat(localCluster.getSkippedShards(), equalTo(0));
@@ -293,8 +338,12 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() {
         Map<String, Object> testClusterInfo = setupTwoClusters();
         int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
 
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         // a query which matches no remote cluster is not a cross cluster search
-        try (EsqlQueryResponse resp = runQuery("from logs-*,x*:no_such_index* | stats sum (v)")) {
+        try (EsqlQueryResponse resp = runQuery("from logs-*,x*:no_such_index* | stats sum (v)", requestIncludeMeta)) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(1));
@@ -303,12 +352,18 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() {
             assertNotNull(executionInfo);
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(LOCAL_CLUSTER)));
             assertThat(executionInfo.isCrossClusterSearch(), is(false));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
             // since this not a CCS, only the overall took time in the EsqlExecutionInfo matters
             assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
         }
 
         // cluster-foo* matches nothing and so should not be present in the EsqlExecutionInfo
-        try (EsqlQueryResponse resp = runQuery("from logs-*,no_such_index*,cluster-a:no_such_index*,cluster-foo*:* | stats sum (v)")) {
+        try (
+            EsqlQueryResponse resp = runQuery(
+                "from logs-*,no_such_index*,cluster-a:no_such_index*,cluster-foo*:* | stats sum (v)",
+                requestIncludeMeta
+            )
+        ) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(1));
@@ -317,6 +372,7 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() {
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
             assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
@@ -349,8 +405,12 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() {
     public void testCCSExecutionOnSearchesWithLimit0() {
         setupTwoClusters();
 
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
         // Ensure non-cross cluster queries have overall took time
-        try (EsqlQueryResponse resp = runQuery("FROM logs* | LIMIT 0")) {
+        try (EsqlQueryResponse resp = runQuery("FROM logs* | LIMIT 0", requestIncludeMeta)) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(false));
@@ -358,12 +418,13 @@ public void testCCSExecutionOnSearchesWithLimit0() {
         }
 
         // ensure cross-cluster searches have overall took time and correct per-cluster details in EsqlExecutionInfo
-        try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:* | LIMIT 0")) {
+        try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:* | LIMIT 0", requestIncludeMeta)) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
             long overallTookMillis = executionInfo.overallTook().millis();
             assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
             EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
@@ -371,34 +432,36 @@ public void testCCSExecutionOnSearchesWithLimit0() {
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
             assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
-            assertNull(remoteCluster.getTotalShards());
-            assertNull(remoteCluster.getSuccessfulShards());
-            assertNull(remoteCluster.getSkippedShards());
-            assertNull(remoteCluster.getFailedShards());
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
             assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
             assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
             assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
-            assertNull(localCluster.getTotalShards());
-            assertNull(localCluster.getSuccessfulShards());
-            assertNull(localCluster.getSkippedShards());
-            assertNull(localCluster.getFailedShards());
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
         }
 
-        try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:nomatch* | LIMIT 0")) {
+        try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:nomatch* | LIMIT 0", requestIncludeMeta)) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
             long overallTookMillis = executionInfo.overallTook().millis();
             assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
             EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
             assertThat(remoteCluster.getIndexExpression(), equalTo("nomatch*"));
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
-            assertThat(remoteCluster.getTook().millis(), equalTo(0L));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(0));
             assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
             assertThat(remoteCluster.getSkippedShards(), equalTo(0));
@@ -409,18 +472,19 @@ public void testCCSExecutionOnSearchesWithLimit0() {
             assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
             assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
-            assertNull(localCluster.getTotalShards());
-            assertNull(localCluster.getSuccessfulShards());
-            assertNull(localCluster.getSkippedShards());
-            assertNull(localCluster.getFailedShards());
+            assertThat(localCluster.getTotalShards(), equalTo(0));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
         }
 
-        try (EsqlQueryResponse resp = runQuery("FROM nomatch*,cluster-a:* | LIMIT 0")) {
+        try (EsqlQueryResponse resp = runQuery("FROM nomatch*,cluster-a:* | LIMIT 0", requestIncludeMeta)) {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
             long overallTookMillis = executionInfo.overallTook().millis();
             assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
 
             EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
@@ -428,17 +492,20 @@ public void testCCSExecutionOnSearchesWithLimit0() {
             assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
             assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
-            assertNull(remoteCluster.getTotalShards());
-            assertNull(remoteCluster.getSuccessfulShards());
-            assertNull(remoteCluster.getSkippedShards());
-            assertNull(remoteCluster.getFailedShards());
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
             assertThat(localCluster.getIndexExpression(), equalTo("nomatch*"));
-            // TODO: in https://github.com/elastic/elasticsearch/issues/112886, this will be changed to be SKIPPED
-            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
             assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
         }
     }
 
@@ -447,7 +514,16 @@ public void testMetadataIndex() {
         int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
         int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
 
-        try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index")) {
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
+        try (
+            EsqlQueryResponse resp = runQuery(
+                "FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index",
+                requestIncludeMeta
+            )
+        ) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values.get(0), equalTo(List.of(285L, "cluster-a:logs-2")));
             assertThat(values.get(1), equalTo(List.of(45L, "logs-1")));
@@ -455,6 +531,7 @@ public void testMetadataIndex() {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
             assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
 
             EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
@@ -477,18 +554,6 @@ public void testMetadataIndex() {
         }
     }
 
-    void waitForNoInitializingShards(Client client, TimeValue timeout, String... indices) {
-        ClusterHealthResponse resp = client.admin()
-            .cluster()
-            .prepareHealth(TEST_REQUEST_TIMEOUT, indices)
-            .setWaitForEvents(Priority.LANGUID)
-            .setWaitForNoRelocatingShards(true)
-            .setWaitForNoInitializingShards(true)
-            .setTimeout(timeout)
-            .get();
-        assertFalse(Strings.toString(resp, true, true), resp.isTimedOut());
-    }
-
     public void testProfile() {
         Map<String, Object> testClusterInfo = setupTwoClusters();
         int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
@@ -529,6 +594,7 @@ public void testProfile() {
                 EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
                 assertNull(remoteCluster);
                 assertThat(executionInfo.isCrossClusterSearch(), is(false));
+                assertThat(executionInfo.includeCCSMetadata(), is(false));
                 // since this not a CCS, only the overall took time in the EsqlExecutionInfo matters
                 assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
             }
@@ -550,6 +616,7 @@ public void testProfile() {
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
                 assertNotNull(executionInfo);
                 assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                assertThat(executionInfo.includeCCSMetadata(), is(false));
                 assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
 
                 EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
@@ -582,6 +649,7 @@ public void testProfile() {
                 EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
                 assertNotNull(executionInfo);
                 assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                assertThat(executionInfo.includeCCSMetadata(), is(false));
                 assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
 
                 EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
@@ -608,14 +676,11 @@ public void testProfile() {
 
     public void testWarnings() throws Exception {
         Map<String, Object> testClusterInfo = setupTwoClusters();
-        String localIndex = (String) testClusterInfo.get("local.index");
-        String remoteIndex = (String) testClusterInfo.get("remote.index");
         int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
         int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
 
         EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
         request.query("FROM logs*,*:logs* | EVAL ip = to_ip(id) | STATS total = sum(v) by ip | LIMIT 10");
-        PlainActionFuture<EsqlQueryResponse> future = new PlainActionFuture<>();
         InternalTestCluster cluster = cluster(LOCAL_CLUSTER);
         String node = randomFrom(cluster.getNodeNames());
         CountDownLatch latch = new CountDownLatch(1);
@@ -634,6 +699,7 @@ public void testWarnings() throws Exception {
             EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
             assertNotNull(executionInfo);
             assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.includeCCSMetadata(), is(false));
             assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
 
             EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
@@ -662,11 +728,34 @@ public void testWarnings() throws Exception {
         assertTrue(latch.await(30, TimeUnit.SECONDS));
     }
 
-    protected EsqlQueryResponse runQuery(String query) {
+    private static void assertClusterMetadataInResponse(EsqlQueryResponse resp, boolean responseExpectMeta) {
+        try {
+            final Map<String, Object> esqlResponseAsMap = XContentTestUtils.convertToMap(resp);
+            final Object clusters = esqlResponseAsMap.get("_clusters");
+            if (responseExpectMeta) {
+                assertNotNull(clusters);
+                // test a few entries to ensure it looks correct (other tests do a full analysis of the metadata in the response)
+                @SuppressWarnings("unchecked")
+                Map<String, Object> inner = (Map<String, Object>) clusters;
+                assertTrue(inner.containsKey("total"));
+                assertTrue(inner.containsKey("details"));
+            } else {
+                assertNull(clusters);
+            }
+        } catch (IOException e) {
+            fail("Could not convert ESQL response to Map: " + e);
+        }
+    }
+
+    protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) {
         EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
         request.query(query);
         request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
-        request.profile(true);
+        request.profile(randomInt(5) == 2);
+        request.columnar(randomBoolean());
+        if (ccsMetadataInResponse != null) {
+            request.includeCCSMetadata(ccsMetadataInResponse);
+        }
         return runQuery(request);
     }
 
@@ -674,6 +763,32 @@ protected EsqlQueryResponse runQuery(EsqlQueryRequest request) {
         return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
     }
 
+    /**
+     * v1: value to send to runQuery (can be null; null means use default value)
+     * v2: whether to expect CCS Metadata in the response (cannot be null)
+     * @return
+     */
+    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
+        return switch (randomIntBetween(1, 3)) {
+            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
+            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
+            case 3 -> new Tuple<>(null, Boolean.FALSE);
+            default -> throw new AssertionError("should not get here");
+        };
+    }
+
+    void waitForNoInitializingShards(Client client, TimeValue timeout, String... indices) {
+        ClusterHealthResponse resp = client.admin()
+            .cluster()
+            .prepareHealth(TEST_REQUEST_TIMEOUT, indices)
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .setTimeout(timeout)
+            .get();
+        assertFalse(Strings.toString(resp, true, true), resp.isTimedOut());
+    }
+
     Map<String, Object> setupTwoClusters() {
         String localIndex = "logs-1";
         int numShardsLocal = randomIntBetween(1, 5);
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java
similarity index 92%
rename from x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java
rename to x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java
index e6f11ca1f44d2..e7da83a40fb20 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java
@@ -14,9 +14,6 @@
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
 import org.elasticsearch.xpack.esql.action.ColumnInfoImpl;
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
-import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
-import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.junit.Before;
 
@@ -29,19 +26,13 @@
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-public class QueryStringFunctionIT extends AbstractEsqlIntegTestCase {
+public class QueryStringIT extends AbstractEsqlIntegTestCase {
 
     @Before
     public void setupIndex() {
         createAndPopulateIndex();
     }
 
-    @Override
-    protected EsqlQueryResponse run(EsqlQueryRequest request) {
-        assumeTrue("qstr function available in snapshot builds only", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
-        return super.run(request);
-    }
-
     public void testSimpleQueryString() {
         var query = """
             FROM test
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java
index 93701552b94aa..94fc4030f73a9 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java
@@ -31,4 +31,10 @@ protected Geometry getQueryGeometry() {
     protected String castingFunction() {
         return "TO_CARTESIANSHAPE";
     }
+
+    @Override
+    protected double searchDistance() {
+        // We search much larger distances for Cartesian, to ensure we actually get results from the much wider data range
+        return 1e12;
+    }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java
index 871fb222de3d4..9bc3312fff63e 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java
@@ -36,4 +36,9 @@ protected Geometry getQueryGeometry() {
     protected String castingFunction() {
         return "TO_GEOSHAPE";
     }
+
+    @Override
+    protected double searchDistance() {
+        return 10000000;
+    }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java
index 411106f008986..0acbe98022f02 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.spatial;
 
+import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.utils.GeometryValidator;
 import org.elasticsearch.geometry.utils.WellKnownText;
@@ -20,6 +21,7 @@
 import java.io.IOException;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Locale;
 
 import static org.hamcrest.Matchers.closeTo;
@@ -108,6 +110,61 @@ protected void assertFunction(String spatialFunction, String wkt, long expected,
         }
     }
 
+    public void testPushedDownDistanceSingleValue() throws RuntimeException {
+        assertPushedDownDistance(false);
+    }
+
+    public void testPushedDownDistanceMultiValue() throws RuntimeException {
+        assertPushedDownDistance(true);
+    }
+
+    private void assertPushedDownDistance(boolean multiValue) throws RuntimeException {
+        initIndexes();
+        for (int i = 0; i < random().nextInt(50, 100); i++) {
+            if (multiValue) {
+                final String[] values = new String[randomIntBetween(1, 5)];
+                for (int j = 0; j < values.length; j++) {
+                    values[j] = "\"" + WellKnownText.toWKT(getIndexGeometry()) + "\"";
+                }
+                index("indexed", i + "", "{\"location\" : " + Arrays.toString(values) + " }");
+                index("not-indexed", i + "", "{\"location\" : " + Arrays.toString(values) + " }");
+            } else {
+                final String value = WellKnownText.toWKT(getIndexGeometry());
+                index("indexed", i + "", "{\"location\" : \"" + value + "\" }");
+                index("not-indexed", i + "", "{\"location\" : \"" + value + "\" }");
+            }
+        }
+
+        refresh("indexed", "not-indexed");
+
+        for (int i = 0; i < 10; i++) {
+            final Geometry geometry = getIndexGeometry();
+            final String wkt = WellKnownText.toWKT(geometry);
+            assertDistanceFunction(wkt);
+        }
+    }
+
+    protected abstract double searchDistance();
+
+    protected void assertDistanceFunction(String wkt) {
+        String spatialFunction = "ST_DISTANCE";
+        String castingFunction = castingFunction().replaceAll("SHAPE", "POINT");
+        final String query1 = String.format(Locale.ROOT, """
+            FROM indexed | WHERE %s(location, %s("%s")) < %.1f | STATS COUNT(*)
+            """, spatialFunction, castingFunction, wkt, searchDistance());
+        final String query2 = String.format(Locale.ROOT, """
+            FROM not-indexed | WHERE %s(location, %s("%s")) < %.1f | STATS COUNT(*)
+            """, spatialFunction, castingFunction, wkt, searchDistance());
+        try (
+            EsqlQueryResponse response1 = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query1).get();
+            EsqlQueryResponse response2 = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query2).get();
+        ) {
+            Object indexedResult = response1.response().column(0).iterator().next();
+            Object notIndexedResult = response2.response().column(0).iterator().next();
+            assertEquals(spatialFunction, indexedResult, notIndexedResult);
+        }
+    }
+
     private String toString(CentroidCalculator centroid) {
         return "Centroid (x:" + centroid.getX() + ", y:" + centroid.getY() + ")";
     }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java
index 9dff647763b6b..e7e0c785f50e5 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java
@@ -28,7 +28,7 @@
 /**
  * Base class to check that a query than can be pushed down gives the same result
  * if it is actually pushed down and when it is executed by the compute engine,
- *
+ * <p>
  * For doing that we create two indices, one fully indexed and another with index
  * and doc values disabled. Then we index the same data in both indices and we check
  * that the same ES|QL queries produce the same results in both.
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
index 0d8d3abf77ecc..ffab261d3c174 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
@@ -86,7 +86,6 @@ WHERE : 'where'               -> pushMode(EXPRESSION_MODE);
 // MYCOMMAND : 'mycommand' -> ...
 DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats'   -> pushMode(EXPRESSION_MODE);
 DEV_LOOKUP :      {this.isDevVersion()}? 'lookup'        -> pushMode(LOOKUP_MODE);
-DEV_MATCH :       {this.isDevVersion()}? 'match'         -> pushMode(EXPRESSION_MODE);
 DEV_METRICS :     {this.isDevVersion()}? 'metrics'       -> pushMode(METRICS_MODE);
 
 //
@@ -209,8 +208,8 @@ ASTERISK : '*';
 SLASH : '/';
 PERCENT : '%';
 
-// move it in the main section if the feature gets promoted
-DEV_MATCH_OP : {this.isDevVersion()}? DEV_MATCH -> type(DEV_MATCH);
+MATCH : 'match';
+NESTED_WHERE : WHERE -> type(WHERE);
 
 NAMED_OR_POSITIONAL_PARAM
     : PARAM (LETTER | UNDERSCORE) UNQUOTED_ID_BODY*
@@ -306,8 +305,8 @@ mode PROJECT_MODE;
 PROJECT_PIPE : PIPE -> type(PIPE), popMode;
 PROJECT_DOT: DOT -> type(DOT);
 PROJECT_COMMA : COMMA -> type(COMMA);
-PROJECT_PARAM : PARAM -> type(PARAM);
-PROJECT_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+PROJECT_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
+PROJECT_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 fragment UNQUOTED_ID_BODY_WITH_PATTERN
     : (LETTER | DIGIT | UNDERSCORE | ASTERISK)
@@ -341,8 +340,8 @@ RENAME_PIPE : PIPE -> type(PIPE), popMode;
 RENAME_ASSIGN : ASSIGN -> type(ASSIGN);
 RENAME_COMMA : COMMA -> type(COMMA);
 RENAME_DOT: DOT -> type(DOT);
-RENAME_PARAM : PARAM -> type(PARAM);
-RENAME_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+RENAME_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
+RENAME_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 AS : 'as';
 
@@ -414,8 +413,8 @@ ENRICH_FIELD_QUOTED_IDENTIFIER
     : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER)
     ;
 
-ENRICH_FIELD_PARAM : PARAM -> type(PARAM);
-ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+ENRICH_FIELD_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
+ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 ENRICH_FIELD_LINE_COMMENT
     : LINE_COMMENT -> channel(HIDDEN)
@@ -432,8 +431,8 @@ ENRICH_FIELD_WS
 mode MVEXPAND_MODE;
 MVEXPAND_PIPE : PIPE -> type(PIPE), popMode;
 MVEXPAND_DOT: DOT -> type(DOT);
-MVEXPAND_PARAM : PARAM -> type(PARAM);
-MVEXPAND_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+MVEXPAND_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM);
+MVEXPAND_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
 
 MVEXPAND_QUOTED_IDENTIFIER
     : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER)
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
index 4fd37ab9900f2..4d1f426289149 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
@@ -16,51 +16,51 @@ STATS=15
 WHERE=16
 DEV_INLINESTATS=17
 DEV_LOOKUP=18
-DEV_MATCH=19
-DEV_METRICS=20
-UNKNOWN_CMD=21
-LINE_COMMENT=22
-MULTILINE_COMMENT=23
-WS=24
-PIPE=25
-QUOTED_STRING=26
-INTEGER_LITERAL=27
-DECIMAL_LITERAL=28
-BY=29
-AND=30
-ASC=31
-ASSIGN=32
-CAST_OP=33
-COMMA=34
-DESC=35
-DOT=36
-FALSE=37
-FIRST=38
-IN=39
-IS=40
-LAST=41
-LIKE=42
-LP=43
-NOT=44
-NULL=45
-NULLS=46
-OR=47
-PARAM=48
-RLIKE=49
-RP=50
-TRUE=51
-EQ=52
-CIEQ=53
-NEQ=54
-LT=55
-LTE=56
-GT=57
-GTE=58
-PLUS=59
-MINUS=60
-ASTERISK=61
-SLASH=62
-PERCENT=63
+DEV_METRICS=19
+UNKNOWN_CMD=20
+LINE_COMMENT=21
+MULTILINE_COMMENT=22
+WS=23
+PIPE=24
+QUOTED_STRING=25
+INTEGER_LITERAL=26
+DECIMAL_LITERAL=27
+BY=28
+AND=29
+ASC=30
+ASSIGN=31
+CAST_OP=32
+COMMA=33
+DESC=34
+DOT=35
+FALSE=36
+FIRST=37
+IN=38
+IS=39
+LAST=40
+LIKE=41
+LP=42
+NOT=43
+NULL=44
+NULLS=45
+OR=46
+PARAM=47
+RLIKE=48
+RP=49
+TRUE=50
+EQ=51
+CIEQ=52
+NEQ=53
+LT=54
+LTE=55
+GT=56
+GTE=57
+PLUS=58
+MINUS=59
+ASTERISK=60
+SLASH=61
+PERCENT=62
+MATCH=63
 NAMED_OR_POSITIONAL_PARAM=64
 OPENING_BRACKET=65
 CLOSING_BRACKET=66
@@ -134,42 +134,43 @@ CLOSING_METRICS_WS=120
 'sort'=14
 'stats'=15
 'where'=16
-'|'=25
-'by'=29
-'and'=30
-'asc'=31
-'='=32
-'::'=33
-','=34
-'desc'=35
-'.'=36
-'false'=37
-'first'=38
-'in'=39
-'is'=40
-'last'=41
-'like'=42
-'('=43
-'not'=44
-'null'=45
-'nulls'=46
-'or'=47
-'?'=48
-'rlike'=49
-')'=50
-'true'=51
-'=='=52
-'=~'=53
-'!='=54
-'<'=55
-'<='=56
-'>'=57
-'>='=58
-'+'=59
-'-'=60
-'*'=61
-'/'=62
-'%'=63
+'|'=24
+'by'=28
+'and'=29
+'asc'=30
+'='=31
+'::'=32
+','=33
+'desc'=34
+'.'=35
+'false'=36
+'first'=37
+'in'=38
+'is'=39
+'last'=40
+'like'=41
+'('=42
+'not'=43
+'null'=44
+'nulls'=45
+'or'=46
+'?'=47
+'rlike'=48
+')'=49
+'true'=50
+'=='=51
+'=~'=52
+'!='=53
+'<'=54
+'<='=55
+'>'=56
+'>='=57
+'+'=58
+'-'=59
+'*'=60
+'/'=61
+'%'=62
+'match'=63
 ']'=66
 'metadata'=75
 'as'=84
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
index b720ba98babf0..f9f994f4ab329 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
@@ -77,7 +77,7 @@ regexBooleanExpression
     ;
 
 matchBooleanExpression
-    : valueExpression DEV_MATCH queryString=string
+    : valueExpression MATCH queryString=string
     ;
 
 valueExpression
@@ -101,7 +101,13 @@ primaryExpression
     ;
 
 functionExpression
-    : identifierOrParameter LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP
+    : functionName LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP
+    ;
+
+functionName
+    // Additional function identifiers that are already a reserved word in the language
+    : MATCH
+    | identifierOrParameter
     ;
 
 dataType
@@ -117,8 +123,7 @@ fields
     ;
 
 field
-    : booleanExpression
-    | qualifiedName ASSIGN booleanExpression
+    : (qualifiedName ASSIGN)? booleanExpression
     ;
 
 fromCommand
@@ -126,8 +131,7 @@ fromCommand
     ;
 
 indexPattern
-    : clusterString COLON indexString
-    | indexString
+    : (clusterString COLON)? indexString
     ;
 
 clusterString
@@ -153,7 +157,7 @@ deprecated_metadata
     ;
 
 metricsCommand
-    : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=fields? (BY grouping=fields)?
+    : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=aggFields? (BY grouping=fields)?
     ;
 
 evalCommand
@@ -161,7 +165,15 @@ evalCommand
     ;
 
 statsCommand
-    : STATS stats=fields? (BY grouping=fields)?
+    : STATS stats=aggFields? (BY grouping=fields)?
+    ;
+
+aggFields
+    : aggField (COMMA aggField)*
+    ;
+
+aggField
+    : field (WHERE booleanExpression)?
     ;
 
 qualifiedName
@@ -183,7 +195,7 @@ identifier
 
 identifierPattern
     : ID_PATTERN
-    | parameter
+    | {this.isDevVersion()}? parameter
     ;
 
 constant
@@ -206,7 +218,7 @@ parameter
 
 identifierOrParameter
     : identifier
-    | parameter
+    | {this.isDevVersion()}? parameter
     ;
 
 limitCommand
@@ -310,5 +322,5 @@ lookupCommand
     ;
 
 inlinestatsCommand
-    : DEV_INLINESTATS stats=fields (BY grouping=fields)?
+    : DEV_INLINESTATS stats=aggFields (BY grouping=fields)?
     ;
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
index 4fd37ab9900f2..4d1f426289149 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
@@ -16,51 +16,51 @@ STATS=15
 WHERE=16
 DEV_INLINESTATS=17
 DEV_LOOKUP=18
-DEV_MATCH=19
-DEV_METRICS=20
-UNKNOWN_CMD=21
-LINE_COMMENT=22
-MULTILINE_COMMENT=23
-WS=24
-PIPE=25
-QUOTED_STRING=26
-INTEGER_LITERAL=27
-DECIMAL_LITERAL=28
-BY=29
-AND=30
-ASC=31
-ASSIGN=32
-CAST_OP=33
-COMMA=34
-DESC=35
-DOT=36
-FALSE=37
-FIRST=38
-IN=39
-IS=40
-LAST=41
-LIKE=42
-LP=43
-NOT=44
-NULL=45
-NULLS=46
-OR=47
-PARAM=48
-RLIKE=49
-RP=50
-TRUE=51
-EQ=52
-CIEQ=53
-NEQ=54
-LT=55
-LTE=56
-GT=57
-GTE=58
-PLUS=59
-MINUS=60
-ASTERISK=61
-SLASH=62
-PERCENT=63
+DEV_METRICS=19
+UNKNOWN_CMD=20
+LINE_COMMENT=21
+MULTILINE_COMMENT=22
+WS=23
+PIPE=24
+QUOTED_STRING=25
+INTEGER_LITERAL=26
+DECIMAL_LITERAL=27
+BY=28
+AND=29
+ASC=30
+ASSIGN=31
+CAST_OP=32
+COMMA=33
+DESC=34
+DOT=35
+FALSE=36
+FIRST=37
+IN=38
+IS=39
+LAST=40
+LIKE=41
+LP=42
+NOT=43
+NULL=44
+NULLS=45
+OR=46
+PARAM=47
+RLIKE=48
+RP=49
+TRUE=50
+EQ=51
+CIEQ=52
+NEQ=53
+LT=54
+LTE=55
+GT=56
+GTE=57
+PLUS=58
+MINUS=59
+ASTERISK=60
+SLASH=61
+PERCENT=62
+MATCH=63
 NAMED_OR_POSITIONAL_PARAM=64
 OPENING_BRACKET=65
 CLOSING_BRACKET=66
@@ -134,42 +134,43 @@ CLOSING_METRICS_WS=120
 'sort'=14
 'stats'=15
 'where'=16
-'|'=25
-'by'=29
-'and'=30
-'asc'=31
-'='=32
-'::'=33
-','=34
-'desc'=35
-'.'=36
-'false'=37
-'first'=38
-'in'=39
-'is'=40
-'last'=41
-'like'=42
-'('=43
-'not'=44
-'null'=45
-'nulls'=46
-'or'=47
-'?'=48
-'rlike'=49
-')'=50
-'true'=51
-'=='=52
-'=~'=53
-'!='=54
-'<'=55
-'<='=56
-'>'=57
-'>='=58
-'+'=59
-'-'=60
-'*'=61
-'/'=62
-'%'=63
+'|'=24
+'by'=28
+'and'=29
+'asc'=30
+'='=31
+'::'=32
+','=33
+'desc'=34
+'.'=35
+'false'=36
+'first'=37
+'in'=38
+'is'=39
+'last'=40
+'like'=41
+'('=42
+'not'=43
+'null'=44
+'nulls'=45
+'or'=46
+'?'=47
+'rlike'=48
+')'=49
+'true'=50
+'=='=51
+'=~'=52
+'!='=53
+'<'=54
+'<='=55
+'>'=56
+'>='=57
+'+'=58
+'-'=59
+'*'=60
+'/'=61
+'%'=62
+'match'=63
 ']'=66
 'metadata'=75
 'as'=84
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java
new file mode 100644
index 0000000000000..22094f7e623e6
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java
@@ -0,0 +1,146 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.DoubleVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.compute.operator.Warnings;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Hypot}.
+ * This class is generated. Do not edit it.
+ */
+public final class HypotEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Source source;
+
+  private final EvalOperator.ExpressionEvaluator n1;
+
+  private final EvalOperator.ExpressionEvaluator n2;
+
+  private final DriverContext driverContext;
+
+  private Warnings warnings;
+
+  public HypotEvaluator(Source source, EvalOperator.ExpressionEvaluator n1,
+      EvalOperator.ExpressionEvaluator n2, DriverContext driverContext) {
+    this.source = source;
+    this.n1 = n1;
+    this.n2 = n2;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (DoubleBlock n1Block = (DoubleBlock) n1.eval(page)) {
+      try (DoubleBlock n2Block = (DoubleBlock) n2.eval(page)) {
+        DoubleVector n1Vector = n1Block.asVector();
+        if (n1Vector == null) {
+          return eval(page.getPositionCount(), n1Block, n2Block);
+        }
+        DoubleVector n2Vector = n2Block.asVector();
+        if (n2Vector == null) {
+          return eval(page.getPositionCount(), n1Block, n2Block);
+        }
+        return eval(page.getPositionCount(), n1Vector, n2Vector).asBlock();
+      }
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, DoubleBlock n1Block, DoubleBlock n2Block) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        if (n1Block.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (n1Block.getValueCount(p) != 1) {
+          if (n1Block.getValueCount(p) > 1) {
+            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (n2Block.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (n2Block.getValueCount(p) != 1) {
+          if (n2Block.getValueCount(p) > 1) {
+            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        result.appendDouble(Hypot.process(n1Block.getDouble(n1Block.getFirstValueIndex(p)), n2Block.getDouble(n2Block.getFirstValueIndex(p))));
+      }
+      return result.build();
+    }
+  }
+
+  public DoubleVector eval(int positionCount, DoubleVector n1Vector, DoubleVector n2Vector) {
+    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        result.appendDouble(p, Hypot.process(n1Vector.getDouble(p), n2Vector.getDouble(p)));
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "HypotEvaluator[" + "n1=" + n1 + ", n2=" + n2 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(n1, n2);
+  }
+
+  private Warnings warnings() {
+    if (warnings == null) {
+      this.warnings = Warnings.createWarnings(
+              driverContext.warningsMode(),
+              source.source().getLineNumber(),
+              source.source().getColumnNumber(),
+              source.text()
+          );
+    }
+    return warnings;
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory n1;
+
+    private final EvalOperator.ExpressionEvaluator.Factory n2;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory n1,
+        EvalOperator.ExpressionEvaluator.Factory n2) {
+      this.source = source;
+      this.n1 = n1;
+      this.n2 = n2;
+    }
+
+    @Override
+    public HypotEvaluator get(DriverContext context) {
+      return new HypotEvaluator(source, n1.get(context), n2.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "HypotEvaluator[" + "n1=" + n1 + ", n2=" + n2 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java
index a3fc2fadbf227..4917b71464dce 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.LongBlock;
-import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
@@ -26,63 +25,42 @@
 public final class StDistanceCartesianPointDocValuesAndConstantEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final Point rightValue;
+  private final Point right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public StDistanceCartesianPointDocValuesAndConstantEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) {
+      EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) {
-      LongVector leftValueVector = leftValueBlock.asVector();
-      if (leftValueVector == null) {
-        return eval(page.getPositionCount(), leftValueBlock);
-      }
-      return eval(page.getPositionCount(), leftValueVector);
+    try (LongBlock leftBlock = (LongBlock) left.eval(page)) {
+      return eval(page.getPositionCount(), leftBlock);
     }
   }
 
-  public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) {
+  public DoubleBlock eval(int positionCount, LongBlock leftBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
-        } catch (IllegalArgumentException e) {
-          warnings().registerException(e);
-          result.appendNull();
-        }
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleBlock eval(int positionCount, LongVector leftValueVector) {
-    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      position: for (int p = 0; p < positionCount; p++) {
-        try {
-          result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
+          StDistance.processCartesianPointDocValuesAndConstant(result, p, leftBlock, this.right);
         } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
@@ -94,12 +72,12 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector) {
 
   @Override
   public String toString() {
-    return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue);
+    Releasables.closeExpectNoException(left);
   }
 
   private Warnings warnings() {
@@ -117,25 +95,24 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final Point rightValue;
+    private final Point right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        Point rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceCartesianPointDocValuesAndConstantEvaluator get(DriverContext context) {
-      return new StDistanceCartesianPointDocValuesAndConstantEvaluator(source, leftValue.get(context), rightValue, context);
+      return new StDistanceCartesianPointDocValuesAndConstantEvaluator(source, left.get(context), right, context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java
index 1da164436a2cb..03c8ba1a04ab6 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java
@@ -4,17 +4,12 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.function.scalar.spatial;
 
-import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
-import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
-import org.elasticsearch.compute.data.DoubleVector;
 import org.elasticsearch.compute.data.LongBlock;
-import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
@@ -29,79 +24,47 @@
 public final class StDistanceCartesianPointDocValuesAndSourceEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final EvalOperator.ExpressionEvaluator rightValue;
+  private final EvalOperator.ExpressionEvaluator right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public StDistanceCartesianPointDocValuesAndSourceEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
+      EvalOperator.ExpressionEvaluator left, EvalOperator.ExpressionEvaluator right,
       DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) {
-      try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) {
-        LongVector leftValueVector = leftValueBlock.asVector();
-        if (leftValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        BytesRefVector rightValueVector = rightValueBlock.asVector();
-        if (rightValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        return eval(page.getPositionCount(), leftValueVector, rightValueVector).asBlock();
+    try (LongBlock leftBlock = (LongBlock) left.eval(page)) {
+      try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) {
+        return eval(page.getPositionCount(), leftBlock, rightBlock);
       }
     }
   }
 
-  public DoubleBlock eval(int positionCount, LongBlock leftValueBlock,
-      BytesRefBlock rightValueBlock) {
+  public DoubleBlock eval(int positionCount, LongBlock leftBlock, BytesRefBlock rightBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef rightValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
+        if (!rightBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (rightValueBlock.isNull(p)) {
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
-        if (rightValueBlock.getValueCount(p) != 1) {
-          if (rightValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
-        }
-        result.appendDouble(StDistance.processCartesianPointDocValuesAndSource(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch)));
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleVector eval(int positionCount, LongVector leftValueVector,
-      BytesRefVector rightValueVector) {
-    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
-      BytesRef rightValueScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
-        result.appendDouble(p, StDistance.processCartesianPointDocValuesAndSource(leftValueVector.getLong(p), rightValueVector.getBytesRef(p, rightValueScratch)));
+        StDistance.processCartesianPointDocValuesAndSource(result, p, leftBlock, rightBlock);
       }
       return result.build();
     }
@@ -109,12 +72,12 @@ public DoubleVector eval(int positionCount, LongVector leftValueVector,
 
   @Override
   public String toString() {
-    return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue, rightValue);
+    Releasables.closeExpectNoException(left, right);
   }
 
   private Warnings warnings() {
@@ -132,25 +95,25 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final EvalOperator.ExpressionEvaluator.Factory rightValue;
+    private final EvalOperator.ExpressionEvaluator.Factory right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        EvalOperator.ExpressionEvaluator.Factory rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left,
+        EvalOperator.ExpressionEvaluator.Factory right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceCartesianPointDocValuesAndSourceEvaluator get(DriverContext context) {
-      return new StDistanceCartesianPointDocValuesAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context);
+      return new StDistanceCartesianPointDocValuesAndSourceEvaluator(source, left.get(context), right.get(context), context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java
index 61329ad7606d0..1085f71e95b73 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java
@@ -4,14 +4,11 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.function.scalar.spatial;
 
-import java.io.IOException;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
-import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -28,66 +25,43 @@
 public final class StDistanceCartesianSourceAndConstantEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final Point rightValue;
+  private final Point right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public StDistanceCartesianSourceAndConstantEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) {
+      EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) {
-      BytesRefVector leftValueVector = leftValueBlock.asVector();
-      if (leftValueVector == null) {
-        return eval(page.getPositionCount(), leftValueBlock);
-      }
-      return eval(page.getPositionCount(), leftValueVector);
+    try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) {
+      return eval(page.getPositionCount(), leftBlock);
     }
   }
 
-  public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
+  public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
-        } catch (IllegalArgumentException | IOException e) {
-          warnings().registerException(e);
-          result.appendNull();
-        }
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) {
-    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
-        try {
-          result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
-        } catch (IllegalArgumentException | IOException e) {
+          StDistance.processCartesianSourceAndConstant(result, p, leftBlock, this.right);
+        } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
         }
@@ -98,12 +72,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) {
 
   @Override
   public String toString() {
-    return "StDistanceCartesianSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceCartesianSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue);
+    Releasables.closeExpectNoException(left);
   }
 
   private Warnings warnings() {
@@ -121,25 +95,24 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final Point rightValue;
+    private final Point right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        Point rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceCartesianSourceAndConstantEvaluator get(DriverContext context) {
-      return new StDistanceCartesianSourceAndConstantEvaluator(source, leftValue.get(context), rightValue, context);
+      return new StDistanceCartesianSourceAndConstantEvaluator(source, left.get(context), right, context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceCartesianSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceCartesianSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java
index c18c9a56fa77a..c8554a3041c89 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java
@@ -4,14 +4,11 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.function.scalar.spatial;
 
-import java.io.IOException;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
-import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -27,88 +24,49 @@
 public final class StDistanceCartesianSourceAndSourceEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final EvalOperator.ExpressionEvaluator rightValue;
+  private final EvalOperator.ExpressionEvaluator right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public StDistanceCartesianSourceAndSourceEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
+      EvalOperator.ExpressionEvaluator left, EvalOperator.ExpressionEvaluator right,
       DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) {
-      try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) {
-        BytesRefVector leftValueVector = leftValueBlock.asVector();
-        if (leftValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        BytesRefVector rightValueVector = rightValueBlock.asVector();
-        if (rightValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        return eval(page.getPositionCount(), leftValueVector, rightValueVector);
+    try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) {
+      try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) {
+        return eval(page.getPositionCount(), leftBlock, rightBlock);
       }
     }
   }
 
-  public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock,
-      BytesRefBlock rightValueBlock) {
+  public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock, BytesRefBlock rightBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
-      BytesRef rightValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
-        }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (rightValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        if (!rightBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (rightValueBlock.getValueCount(p) != 1) {
-          if (rightValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processCartesianSourceAndSource(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch)));
-        } catch (IllegalArgumentException | IOException e) {
-          warnings().registerException(e);
-          result.appendNull();
-        }
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector,
-      BytesRefVector rightValueVector) {
-    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
-      BytesRef rightValueScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
-        try {
-          result.appendDouble(StDistance.processCartesianSourceAndSource(leftValueVector.getBytesRef(p, leftValueScratch), rightValueVector.getBytesRef(p, rightValueScratch)));
-        } catch (IllegalArgumentException | IOException e) {
+          StDistance.processCartesianSourceAndSource(result, p, leftBlock, rightBlock);
+        } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
         }
@@ -119,12 +77,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector,
 
   @Override
   public String toString() {
-    return "StDistanceCartesianSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceCartesianSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue, rightValue);
+    Releasables.closeExpectNoException(left, right);
   }
 
   private Warnings warnings() {
@@ -142,25 +100,25 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final EvalOperator.ExpressionEvaluator.Factory rightValue;
+    private final EvalOperator.ExpressionEvaluator.Factory right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        EvalOperator.ExpressionEvaluator.Factory rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left,
+        EvalOperator.ExpressionEvaluator.Factory right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceCartesianSourceAndSourceEvaluator get(DriverContext context) {
-      return new StDistanceCartesianSourceAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context);
+      return new StDistanceCartesianSourceAndSourceEvaluator(source, left.get(context), right.get(context), context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceCartesianSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceCartesianSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java
index 2ac1ff6aeb0d8..8f37d3157fac6 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.LongBlock;
-import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
@@ -26,63 +25,42 @@
 public final class StDistanceGeoPointDocValuesAndConstantEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final Point rightValue;
+  private final Point right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public StDistanceGeoPointDocValuesAndConstantEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) {
+      EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) {
-      LongVector leftValueVector = leftValueBlock.asVector();
-      if (leftValueVector == null) {
-        return eval(page.getPositionCount(), leftValueBlock);
-      }
-      return eval(page.getPositionCount(), leftValueVector);
+    try (LongBlock leftBlock = (LongBlock) left.eval(page)) {
+      return eval(page.getPositionCount(), leftBlock);
     }
   }
 
-  public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) {
+  public DoubleBlock eval(int positionCount, LongBlock leftBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
-        } catch (IllegalArgumentException e) {
-          warnings().registerException(e);
-          result.appendNull();
-        }
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleBlock eval(int positionCount, LongVector leftValueVector) {
-    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      position: for (int p = 0; p < positionCount; p++) {
-        try {
-          result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
+          StDistance.processGeoPointDocValuesAndConstant(result, p, leftBlock, this.right);
         } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
@@ -94,12 +72,12 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector) {
 
   @Override
   public String toString() {
-    return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue);
+    Releasables.closeExpectNoException(left);
   }
 
   private Warnings warnings() {
@@ -117,25 +95,24 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final Point rightValue;
+    private final Point right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        Point rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceGeoPointDocValuesAndConstantEvaluator get(DriverContext context) {
-      return new StDistanceGeoPointDocValuesAndConstantEvaluator(source, leftValue.get(context), rightValue, context);
+      return new StDistanceGeoPointDocValuesAndConstantEvaluator(source, left.get(context), right, context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java
index 6758d888cc7d1..a7664987739e2 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java
@@ -7,13 +7,10 @@
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
-import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.LongBlock;
-import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
@@ -28,85 +25,48 @@
 public final class StDistanceGeoPointDocValuesAndSourceEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final EvalOperator.ExpressionEvaluator rightValue;
+  private final EvalOperator.ExpressionEvaluator right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public StDistanceGeoPointDocValuesAndSourceEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
+      EvalOperator.ExpressionEvaluator left, EvalOperator.ExpressionEvaluator right,
       DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) {
-      try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) {
-        LongVector leftValueVector = leftValueBlock.asVector();
-        if (leftValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        BytesRefVector rightValueVector = rightValueBlock.asVector();
-        if (rightValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        return eval(page.getPositionCount(), leftValueVector, rightValueVector);
+    try (LongBlock leftBlock = (LongBlock) left.eval(page)) {
+      try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) {
+        return eval(page.getPositionCount(), leftBlock, rightBlock);
       }
     }
   }
 
-  public DoubleBlock eval(int positionCount, LongBlock leftValueBlock,
-      BytesRefBlock rightValueBlock) {
+  public DoubleBlock eval(int positionCount, LongBlock leftBlock, BytesRefBlock rightBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef rightValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
+        if (!rightBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (rightValueBlock.isNull(p)) {
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
-        if (rightValueBlock.getValueCount(p) != 1) {
-          if (rightValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
-        }
-        try {
-          result.appendDouble(StDistance.processGeoPointDocValuesAndSource(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch)));
-        } catch (IllegalArgumentException e) {
-          warnings().registerException(e);
-          result.appendNull();
-        }
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleBlock eval(int positionCount, LongVector leftValueVector,
-      BytesRefVector rightValueVector) {
-    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef rightValueScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendDouble(StDistance.processGeoPointDocValuesAndSource(leftValueVector.getLong(p), rightValueVector.getBytesRef(p, rightValueScratch)));
+          StDistance.processGeoPointDocValuesAndSource(result, p, leftBlock, rightBlock);
         } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
@@ -118,12 +78,12 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector,
 
   @Override
   public String toString() {
-    return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue, rightValue);
+    Releasables.closeExpectNoException(left, right);
   }
 
   private Warnings warnings() {
@@ -141,25 +101,25 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final EvalOperator.ExpressionEvaluator.Factory rightValue;
+    private final EvalOperator.ExpressionEvaluator.Factory right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        EvalOperator.ExpressionEvaluator.Factory rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left,
+        EvalOperator.ExpressionEvaluator.Factory right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceGeoPointDocValuesAndSourceEvaluator get(DriverContext context) {
-      return new StDistanceGeoPointDocValuesAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context);
+      return new StDistanceGeoPointDocValuesAndSourceEvaluator(source, left.get(context), right.get(context), context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java
index 201c29cb04275..06e44f996daf5 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java
@@ -4,14 +4,11 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.function.scalar.spatial;
 
-import java.io.IOException;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
-import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -28,66 +25,43 @@
 public final class StDistanceGeoSourceAndConstantEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final Point rightValue;
+  private final Point right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public StDistanceGeoSourceAndConstantEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) {
+      EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) {
-      BytesRefVector leftValueVector = leftValueBlock.asVector();
-      if (leftValueVector == null) {
-        return eval(page.getPositionCount(), leftValueBlock);
-      }
-      return eval(page.getPositionCount(), leftValueVector);
+    try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) {
+      return eval(page.getPositionCount(), leftBlock);
     }
   }
 
-  public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
+  public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
-        } catch (IllegalArgumentException | IOException e) {
-          warnings().registerException(e);
-          result.appendNull();
-        }
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) {
-    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
-        try {
-          result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
-        } catch (IllegalArgumentException | IOException e) {
+          StDistance.processGeoSourceAndConstant(result, p, leftBlock, this.right);
+        } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
         }
@@ -98,12 +72,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) {
 
   @Override
   public String toString() {
-    return "StDistanceGeoSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceGeoSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue);
+    Releasables.closeExpectNoException(left);
   }
 
   private Warnings warnings() {
@@ -121,25 +95,24 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final Point rightValue;
+    private final Point right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        Point rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceGeoSourceAndConstantEvaluator get(DriverContext context) {
-      return new StDistanceGeoSourceAndConstantEvaluator(source, leftValue.get(context), rightValue, context);
+      return new StDistanceGeoSourceAndConstantEvaluator(source, left.get(context), right, context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceGeoSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceGeoSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java
index 9cbe30e22c3ed..31e20d9f42197 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java
@@ -4,14 +4,11 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.function.scalar.spatial;
 
-import java.io.IOException;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
-import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -27,88 +24,48 @@
 public final class StDistanceGeoSourceAndSourceEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator leftValue;
+  private final EvalOperator.ExpressionEvaluator left;
 
-  private final EvalOperator.ExpressionEvaluator rightValue;
+  private final EvalOperator.ExpressionEvaluator right;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
-  public StDistanceGeoSourceAndSourceEvaluator(Source source,
-      EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
-      DriverContext driverContext) {
+  public StDistanceGeoSourceAndSourceEvaluator(Source source, EvalOperator.ExpressionEvaluator left,
+      EvalOperator.ExpressionEvaluator right, DriverContext driverContext) {
     this.source = source;
-    this.leftValue = leftValue;
-    this.rightValue = rightValue;
+    this.left = left;
+    this.right = right;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) {
-      try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) {
-        BytesRefVector leftValueVector = leftValueBlock.asVector();
-        if (leftValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        BytesRefVector rightValueVector = rightValueBlock.asVector();
-        if (rightValueVector == null) {
-          return eval(page.getPositionCount(), leftValueBlock, rightValueBlock);
-        }
-        return eval(page.getPositionCount(), leftValueVector, rightValueVector);
+    try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) {
+      try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) {
+        return eval(page.getPositionCount(), leftBlock, rightBlock);
       }
     }
   }
 
-  public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock,
-      BytesRefBlock rightValueBlock) {
+  public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock, BytesRefBlock rightBlock) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
-      BytesRef rightValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        if (leftValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
-        }
-        if (leftValueBlock.getValueCount(p) != 1) {
-          if (leftValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
+        boolean allBlocksAreNulls = true;
+        if (!leftBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (rightValueBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
+        if (!rightBlock.isNull(p)) {
+          allBlocksAreNulls = false;
         }
-        if (rightValueBlock.getValueCount(p) != 1) {
-          if (rightValueBlock.getValueCount(p) > 1) {
-            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
+        if (allBlocksAreNulls) {
           result.appendNull();
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processGeoSourceAndSource(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch)));
-        } catch (IllegalArgumentException | IOException e) {
-          warnings().registerException(e);
-          result.appendNull();
-        }
-      }
-      return result.build();
-    }
-  }
-
-  public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector,
-      BytesRefVector rightValueVector) {
-    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
-      BytesRef leftValueScratch = new BytesRef();
-      BytesRef rightValueScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
-        try {
-          result.appendDouble(StDistance.processGeoSourceAndSource(leftValueVector.getBytesRef(p, leftValueScratch), rightValueVector.getBytesRef(p, rightValueScratch)));
-        } catch (IllegalArgumentException | IOException e) {
+          StDistance.processGeoSourceAndSource(result, p, leftBlock, rightBlock);
+        } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
         }
@@ -119,12 +76,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector,
 
   @Override
   public String toString() {
-    return "StDistanceGeoSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+    return "StDistanceGeoSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(leftValue, rightValue);
+    Releasables.closeExpectNoException(left, right);
   }
 
   private Warnings warnings() {
@@ -142,25 +99,25 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory leftValue;
+    private final EvalOperator.ExpressionEvaluator.Factory left;
 
-    private final EvalOperator.ExpressionEvaluator.Factory rightValue;
+    private final EvalOperator.ExpressionEvaluator.Factory right;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue,
-        EvalOperator.ExpressionEvaluator.Factory rightValue) {
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left,
+        EvalOperator.ExpressionEvaluator.Factory right) {
       this.source = source;
-      this.leftValue = leftValue;
-      this.rightValue = rightValue;
+      this.left = left;
+      this.right = right;
     }
 
     @Override
     public StDistanceGeoSourceAndSourceEvaluator get(DriverContext context) {
-      return new StDistanceGeoSourceAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context);
+      return new StDistanceGeoSourceAndSourceEvaluator(source, left.get(context), right.get(context), context);
     }
 
     @Override
     public String toString() {
-      return "StDistanceGeoSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]";
+      return "StDistanceGeoSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index 9aa4d874c53e2..55236af648236 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -32,11 +32,23 @@ public enum Cap {
          */
         FN_REVERSE,
 
+        /**
+         * Support for reversing whole grapheme clusters. This is not supported
+         * on JDK versions less than 20 which are not supported in ES 9.0.0+ but this
+         * exists to keep the {@code 8.x} branch similar to the {@code main} branch.
+         */
+        FN_REVERSE_GRAPHEME_CLUSTERS,
+
         /**
          * Support for function {@code CBRT}. Done in #108574.
          */
         FN_CBRT,
 
+        /**
+         * Support for function {@code HYPOT}.
+         */
+        FN_HYPOT,
+
         /**
          * Support for {@code MV_APPEND} function. #107001
          */
@@ -128,7 +140,7 @@ public enum Cap {
          * - fixed variable shadowing
          * - fixed Join.references(), requiring breaking change to Join serialization
          */
-        LOOKUP_V4(true),
+        LOOKUP_V4(Build.current().isSnapshot()),
 
         /**
          * Support for requesting the "REPEAT" command.
@@ -165,11 +177,21 @@ public enum Cap {
          */
         SPATIAL_PREDICATES_SUPPORT_MULTIVALUES,
 
+        /**
+         * Enable spatial distance function to support multi-values. Done in #114836.
+         */
+        SPATIAL_DISTANCE_SUPPORTS_MULTIVALUES,
+
         /**
          * Support a number of fixes and enhancements to spatial distance pushdown. Done in #112938.
          */
         SPATIAL_DISTANCE_PUSHDOWN_ENHANCEMENTS,
 
+        /**
+         * Fix for spatial centroid when no records are found.
+         */
+        SPATIAL_CENTROID_NO_RECORDS,
+
         /**
          * Fix to GROK and DISSECT that allows extracting attributes with the same name as the input
          * https://github.com/elastic/elasticsearch/issues/110184
@@ -264,7 +286,7 @@ public enum Cap {
         /**
          * Support for match operator
          */
-        MATCH_OPERATOR(true),
+        MATCH_OPERATOR(Build.current().isSnapshot()),
 
         /**
          * Removing support for the {@code META} keyword.
@@ -286,6 +308,16 @@ public enum Cap {
          */
         TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
 
+        /**
+         * Support Least and Greatest functions on Date Nanos type
+         */
+        LEAST_GREATEST_FOR_DATENANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+
+        /**
+         * support aggregations on date nanos
+         */
+        DATE_NANOS_AGGREGATIONS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+
         /**
          * Support for datetime in least and greatest functions
          */
@@ -334,12 +366,17 @@ public enum Cap {
         /**
          * Supported the text categorization function "CATEGORIZE".
          */
-        CATEGORIZE(true),
+        CATEGORIZE(Build.current().isSnapshot()),
 
         /**
          * QSTR function
          */
-        QSTR_FUNCTION(true),
+        QSTR_FUNCTION,
+
+        /**
+         * MATCH function
+         */
+        MATCH_FUNCTION,
 
         /**
          * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well.
@@ -353,36 +390,70 @@ public enum Cap {
         DATE_DIFF_YEAR_CALENDARIAL,
 
         /**
-         * Support named parameters for field names.
+         * Fix sorting not allowed on _source and counters.
+         */
+        SORTING_ON_SOURCE_AND_COUNTERS_FORBIDDEN,
+
+        /**
+         * Allow filter per individual aggregation.
          */
-        NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES;
+        PER_AGG_FILTERING,
 
-        private final boolean snapshotOnly;
-        private final FeatureFlag featureFlag;
+        /**
+         * Fix {@link #PER_AGG_FILTERING} grouped by ordinals.
+         */
+        PER_AGG_FILTERING_ORDS,
+
+        /**
+         * Fix for https://github.com/elastic/elasticsearch/issues/114714
+         */
+        FIX_STATS_BY_FOLDABLE_EXPRESSION,
+
+        /**
+         * Adding stats for functions (stack telemetry)
+         */
+        FUNCTION_STATS,
+        /**
+         * Support for semantic_text field mapping
+         */
+        SEMANTIC_TEXT_TYPE(EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG),
+        /**
+         * Fix for an optimization that caused wrong results
+         * https://github.com/elastic/elasticsearch/issues/115281
+         */
+        FIX_FILTER_PUSHDOWN_PAST_STATS,
+
+        /**
+         * This enables 60_usage.yml "Basic ESQL usage....snapshot" version test. See also the next capability.
+         */
+        SNAPSHOT_TEST_FOR_TELEMETRY(Build.current().isSnapshot()),
+
+        /**
+         * This enables 60_usage.yml "Basic ESQL usage....non-snapshot" version test. See also the previous capability.
+         */
+        NON_SNAPSHOT_TEST_FOR_TELEMETRY(Build.current().isSnapshot() == false),
+
+        /**
+         * Support simplified syntax for named parameters for field and function names.
+         */
+        NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX(Build.current().isSnapshot());
+
+        private final boolean enabled;
 
         Cap() {
-            this(false, null);
+            this.enabled = true;
         };
 
-        Cap(boolean snapshotOnly) {
-            this(snapshotOnly, null);
+        Cap(boolean enabled) {
+            this.enabled = enabled;
         };
 
         Cap(FeatureFlag featureFlag) {
-            this(false, featureFlag);
-        }
-
-        Cap(boolean snapshotOnly, FeatureFlag featureFlag) {
-            assert featureFlag == null || snapshotOnly == false;
-            this.snapshotOnly = snapshotOnly;
-            this.featureFlag = featureFlag;
+            this.enabled = featureFlag.isEnabled();
         }
 
         public boolean isEnabled() {
-            if (featureFlag == null) {
-                return Build.current().isSnapshot() || this.snapshotOnly == false;
-            }
-            return featureFlag.isEnabled();
+            return enabled;
         }
 
         public String capabilityName() {
@@ -390,12 +461,17 @@ public String capabilityName() {
         }
     }
 
-    public static final Set<String> CAPABILITIES = capabilities();
+    public static final Set<String> CAPABILITIES = capabilities(false);
 
-    private static Set<String> capabilities() {
+    /**
+     * Get a {@link Set} of all capabilities. If the {@code all} parameter is {@code false}
+     * then only <strong>enabled</strong> capabilities are returned - otherwise <strong>all</strong>
+     * known capabilities are returned.
+     */
+    public static Set<String> capabilities(boolean all) {
         List<String> caps = new ArrayList<>();
         for (Cap cap : Cap.values()) {
-            if (cap.isEnabled()) {
+            if (all || cap.isEnabled()) {
                 caps.add(cap.capabilityName());
             }
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
index f7966ff5ae9ec..aeac14091f378 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.action;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -32,6 +33,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
 
@@ -54,34 +56,44 @@ public class EsqlExecutionInfo implements ChunkedToXContentObject, Writeable {
     public static final ParseField DETAILS_FIELD = new ParseField("details");
     public static final ParseField TOOK = new ParseField("took");
 
-    // map key is clusterAlias on the primary querying cluster of a CCS minimize_roundtrips=true query
-    // the Map itself is immutable after construction - all Clusters will be accounted for at the start of the search
-    // updates to the Cluster occur with the updateCluster method that given the key to map transforms an
+    // Map key is clusterAlias on the primary querying cluster of a CCS minimize_roundtrips=true query
+    // The Map itself is immutable after construction - all Clusters will be accounted for at the start of the search.
+    // Updates to the Cluster occur with the updateCluster method that given the key to map transforms an
     // old Cluster Object to a new Cluster Object with the remapping function.
     public final Map<String, Cluster> clusterInfo;
-    // not Writeable since it is only needed on the primary CCS coordinator
-    private final transient Predicate<String> skipUnavailablePredicate;
     private TimeValue overallTook;
+    // whether the user has asked for CCS metadata to be in the JSON response (the overall took will always be present)
+    private final boolean includeCCSMetadata;
+
+    // fields that are not Writeable since they are only needed on the primary CCS coordinator
+    private final transient Predicate<String> skipUnavailablePredicate;
+    private final transient Long relativeStartNanos;  // start time for an ESQL query for calculating took times
+    private transient TimeValue planningTookTime;  // time elapsed since start of query to calling ComputeService.execute
 
-    public EsqlExecutionInfo() {
-        this(Predicates.always());  // default all clusters to skip_unavailable=true
+    public EsqlExecutionInfo(boolean includeCCSMetadata) {
+        this(Predicates.always(), includeCCSMetadata);  // default all clusters to skip_unavailable=true
     }
 
     /**
      * @param skipUnavailablePredicate provide lookup for whether a given cluster has skip_unavailable set to true or false
+     * @param includeCCSMetadata (user defined setting) whether to include the CCS metadata in the HTTP response
      */
-    public EsqlExecutionInfo(Predicate<String> skipUnavailablePredicate) {
+    public EsqlExecutionInfo(Predicate<String> skipUnavailablePredicate, boolean includeCCSMetadata) {
         this.clusterInfo = ConcurrentCollections.newConcurrentMap();
         this.skipUnavailablePredicate = skipUnavailablePredicate;
+        this.includeCCSMetadata = includeCCSMetadata;
+        this.relativeStartNanos = System.nanoTime();
     }
 
     /**
      * For testing use with fromXContent parsing only
      * @param clusterInfo
      */
-    EsqlExecutionInfo(ConcurrentMap<String, Cluster> clusterInfo) {
+    EsqlExecutionInfo(ConcurrentMap<String, Cluster> clusterInfo, boolean includeCCSMetadata) {
         this.clusterInfo = clusterInfo;
+        this.includeCCSMetadata = includeCCSMetadata;
         this.skipUnavailablePredicate = Predicates.always();
+        this.relativeStartNanos = null;
     }
 
     public EsqlExecutionInfo(StreamInput in) throws IOException {
@@ -94,7 +106,13 @@ public EsqlExecutionInfo(StreamInput in) throws IOException {
             clusterList.forEach(c -> m.put(c.getClusterAlias(), c));
             this.clusterInfo = m;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.OPT_IN_ESQL_CCS_EXECUTION_INFO)) {
+            this.includeCCSMetadata = in.readBoolean();
+        } else {
+            this.includeCCSMetadata = false;
+        }
         this.skipUnavailablePredicate = Predicates.always();
+        this.relativeStartNanos = null;
     }
 
     @Override
@@ -105,9 +123,44 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeCollection(Collections.emptyList());
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.OPT_IN_ESQL_CCS_EXECUTION_INFO)) {
+            out.writeBoolean(includeCCSMetadata);
+        }
+    }
+
+    public boolean includeCCSMetadata() {
+        return includeCCSMetadata;
+    }
+
+    public Long getRelativeStartNanos() {
+        return relativeStartNanos;
+    }
+
+    /**
+     * Call when ES|QL "planning" phase is complete and query execution (in ComputeService) is about to start.
+     * Note this is currently only built for a single phase planning/execution model. When INLINESTATS
+     * moves towards GA we may need to revisit this model. Currently, it should never be called more than once.
+     */
+    public void markEndPlanning() {
+        assert planningTookTime == null : "markEndPlanning should only be called once";
+        assert relativeStartNanos != null : "Relative start time must be set when markEndPlanning is called";
+        planningTookTime = new TimeValue(System.nanoTime() - relativeStartNanos, TimeUnit.NANOSECONDS);
+    }
+
+    public TimeValue planningTookTime() {
+        return planningTookTime;
+    }
+
+    /**
+     * Call when ES|QL execution is complete in order to set the overall took time for an ES|QL query.
+     */
+    public void markEndQuery() {
+        assert relativeStartNanos != null : "Relative start time must be set when markEndQuery is called";
+        overallTook = new TimeValue(System.nanoTime() - relativeStartNanos, TimeUnit.NANOSECONDS);
     }
 
-    public void overallTook(TimeValue took) {
+    // for testing only - use markEndQuery in production code
+    void overallTook(TimeValue took) {
         this.overallTook = took;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java
index 4ab310863c61d..239f9e2696f88 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java
@@ -42,6 +42,7 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E
     private String query;
     private boolean columnar;
     private boolean profile;
+    private boolean includeCCSMetadata;
     private Locale locale;
     private QueryBuilder filter;
     private QueryPragmas pragmas = new QueryPragmas(Settings.EMPTY);
@@ -128,6 +129,14 @@ public void profile(boolean profile) {
         this.profile = profile;
     }
 
+    public void includeCCSMetadata(boolean include) {
+        this.includeCCSMetadata = include;
+    }
+
+    public boolean includeCCSMetadata() {
+        return includeCCSMetadata;
+    }
+
     /**
      * Is profiling enabled?
      */
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
index 3232f3a9118d4..4e59d5419fe6f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
@@ -206,7 +206,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 b.append(ResponseXContentUtils.allColumns(columns, "columns"));
             }
             b.array("values", ResponseXContentUtils.columnValues(this.columns, this.pages, columnar, nullColumns));
-            if (executionInfo != null && executionInfo.isCrossClusterSearch()) {
+            if (executionInfo != null && executionInfo.isCrossClusterSearch() && executionInfo.includeCCSMetadata()) {
                 b.field("_clusters", executionInfo);
             }
             if (profile != null) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java
index 0b1bafdab1a99..0def56c70dc35 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java
@@ -89,7 +89,7 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa
                     return builder.value(unsignedLongAsNumber(l));
                 }
             };
-            case KEYWORD, TEXT -> new PositionToXContent(block) {
+            case KEYWORD, SEMANTIC_TEXT, TEXT -> new PositionToXContent(block) {
                 @Override
                 protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
                     throws IOException {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
index b930fa5823404..d8904288523a7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.esql.action;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -80,6 +79,7 @@ String fields() {
     private static final ParseField LOCALE_FIELD = new ParseField("locale");
     private static final ParseField PROFILE_FIELD = new ParseField("profile");
     private static final ParseField ACCEPT_PRAGMA_RISKS = new ParseField("accept_pragma_risks");
+    private static final ParseField INCLUDE_CCS_METADATA_FIELD = new ParseField("include_ccs_metadata");
     static final ParseField TABLES_FIELD = new ParseField("tables");
 
     static final ParseField WAIT_FOR_COMPLETION_TIMEOUT = new ParseField("wait_for_completion_timeout");
@@ -89,19 +89,6 @@ String fields() {
     private static final ObjectParser<EsqlQueryRequest, Void> SYNC_PARSER = objectParserSync(EsqlQueryRequest::syncEsqlQueryRequest);
     private static final ObjectParser<EsqlQueryRequest, Void> ASYNC_PARSER = objectParserAsync(EsqlQueryRequest::asyncEsqlQueryRequest);
 
-    private enum ParamParsingKey {
-        VALUE,
-        KIND
-    }
-
-    private static final Map<String, ParamParsingKey> paramParsingKeys = Maps.newMapWithExpectedSize(ParamParsingKey.values().length);
-
-    static {
-        for (ParamParsingKey e : ParamParsingKey.values()) {
-            paramParsingKeys.put(e.name(), e);
-        }
-    }
-
     /** Parses a synchronous request. */
     static EsqlQueryRequest parseSync(XContentParser parser) {
         return SYNC_PARSER.apply(parser, null);
@@ -117,6 +104,7 @@ private static void objectParserCommon(ObjectParser<EsqlQueryRequest, ?> parser)
         parser.declareBoolean(EsqlQueryRequest::columnar, COLUMNAR_FIELD);
         parser.declareObject(EsqlQueryRequest::filter, (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p), FILTER_FIELD);
         parser.declareBoolean(EsqlQueryRequest::acceptedPragmaRisks, ACCEPT_PRAGMA_RISKS);
+        parser.declareBoolean(EsqlQueryRequest::includeCCSMetadata, INCLUDE_CCS_METADATA_FIELD);
         parser.declareObject(
             EsqlQueryRequest::pragmas,
             (p, c) -> new QueryPragmas(Settings.builder().loadFromMap(p.map()).build()),
@@ -178,24 +166,21 @@ private static QueryParams parseParams(XContentParser p) throws IOException {
                         );
                     }
                     for (Map.Entry<String, Object> entry : param.fields.entrySet()) {
-                        ParserUtils.ParamClassification classification;
+                        ParserUtils.ParamClassification classification = null;
+                        paramValue = null;
                         String paramName = entry.getKey();
                         checkParamNameValidity(paramName, errors, loc);
 
-                        if (entry.getValue() instanceof Map<?, ?> values) {// parameter specified as key:value pairs
-                            Map<ParamParsingKey, Object> paramElements = Maps.newMapWithExpectedSize(2);
-                            for (Object keyName : values.keySet()) {
-                                ParamParsingKey paramType = checkParamValueKeysValidity(keyName.toString(), errors, loc);
-                                if (paramType != null) {
-                                    paramElements.put(paramType, values.get(keyName));
+                        if (EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+                            && entry.getValue() instanceof Map<?, ?> value) {// parameter specified as a key:value pair
+                            checkParamValueSize(paramName, value, loc, errors);
+                            for (Object keyName : value.keySet()) {
+                                classification = getParamClassification(keyName.toString(), errors, loc);
+                                if (classification != null) {
+                                    paramValue = value.get(keyName);
+                                    checkParamValueValidity(classification, paramValue, loc, errors);
                                 }
                             }
-                            paramValue = paramElements.get(ParamParsingKey.VALUE);
-                            if (paramValue == null && values.size() > 1) { // require non-null value for identifier and pattern
-                                errors.add(new XContentParseException(loc, "[" + entry + "] does not have a value specified"));
-                            }
-
-                            classification = getClassificationForParam(paramElements, loc, errors);
                         } else {// parameter specifies as a value only
                             paramValue = entry.getValue();
                             classification = VALUE;
@@ -277,12 +262,45 @@ private static void checkParamNameValidity(String name, List<XContentParseExcept
         }
     }
 
-    private static ParamParsingKey checkParamValueKeysValidity(
+    private static void checkParamValueSize(
+        String paramName,
+        Map<?, ?> paramValue,
+        XContentLocation loc,
+        List<XContentParseException> errors
+    ) {
+        if (paramValue.size() == 1) {
+            return;
+        }
+        String errorMessage;
+        if (paramValue.isEmpty()) {
+            errorMessage = " has no valid param attribute";
+        } else {
+            errorMessage = " has multiple param attributes ["
+                + paramValue.keySet().stream().map(Object::toString).collect(Collectors.joining(", "))
+                + "]";
+        }
+        errors.add(
+            new XContentParseException(
+                loc,
+                "["
+                    + paramName
+                    + "]"
+                    + errorMessage
+                    + ", only one of "
+                    + Arrays.stream(ParserUtils.ParamClassification.values())
+                        .map(ParserUtils.ParamClassification::name)
+                        .collect(Collectors.joining(", "))
+                    + " can be defined in a param"
+            )
+        );
+    }
+
+    private static ParserUtils.ParamClassification getParamClassification(
         String paramKeyName,
         List<XContentParseException> errors,
         XContentLocation loc
     ) {
-        ParamParsingKey paramType = paramParsingKeys.get(paramKeyName.toUpperCase(Locale.ROOT));
+        ParserUtils.ParamClassification paramType = paramClassifications.get(paramKeyName.toUpperCase(Locale.ROOT));
         if (paramType == null) {
             errors.add(
                 new XContentParseException(
@@ -290,38 +308,21 @@ private static ParamParsingKey checkParamValueKeysValidity(
                     "["
                         + paramKeyName
                         + "] is not a valid param attribute, a valid attribute is any of "
-                        + Arrays.stream(ParamParsingKey.values()).map(ParamParsingKey::name).collect(Collectors.joining(", "))
+                        + Arrays.stream(ParserUtils.ParamClassification.values())
+                            .map(ParserUtils.ParamClassification::name)
+                            .collect(Collectors.joining(", "))
                 )
             );
         }
         return paramType;
     }
 
-    private static ParserUtils.ParamClassification getClassificationForParam(
-        Map<ParamParsingKey, Object> paramElements,
+    private static void checkParamValueValidity(
+        ParserUtils.ParamClassification classification,
+        Object value,
         XContentLocation loc,
         List<XContentParseException> errors
     ) {
-        Object value = paramElements.get(ParamParsingKey.VALUE);
-        Object kind = paramElements.get(ParamParsingKey.KIND);
-        ParserUtils.ParamClassification classification = VALUE;
-        if (kind != null) {
-            classification = paramClassifications.get(kind.toString().toUpperCase(Locale.ROOT));
-            if (classification == null) {
-                errors.add(
-                    new XContentParseException(
-                        loc,
-                        "["
-                            + kind
-                            + "] is not a valid param kind, a valid kind is any of "
-                            + Arrays.stream(ParserUtils.ParamClassification.values())
-                                .map(ParserUtils.ParamClassification::name)
-                                .collect(Collectors.joining(", "))
-                    )
-                );
-            }
-        }
-
         // If a param is an "identifier" or a "pattern", validate it is a string.
         // If a param is a "pattern", validate it contains *.
         if (classification == IDENTIFIER || classification == PATTERN) {
@@ -342,6 +343,5 @@ private static ParserUtils.ParamClassification getClassificationForParam(
                 );
             }
         }
-        return classification;
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java
index 3b18bda120e2e..49fcc167dce0f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java
@@ -114,7 +114,7 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR
             case LONG, COUNTER_LONG -> ((LongBlock) block).getLong(offset);
             case INTEGER, COUNTER_INTEGER -> ((IntBlock) block).getInt(offset);
             case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock) block).getDouble(offset);
-            case KEYWORD, TEXT -> ((BytesRefBlock) block).getBytesRef(offset, scratch).utf8ToString();
+            case KEYWORD, SEMANTIC_TEXT, TEXT -> ((BytesRefBlock) block).getBytesRef(offset, scratch).utf8ToString();
             case IP -> {
                 BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch);
                 yield ipToString(val);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index 90957f55141b9..b18f58b0a43cb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -228,13 +228,13 @@ public static List<Attribute> mappingAsAttributes(Source source, Map<String, EsF
         return list;
     }
 
-    private static void mappingAsAttributes(List<Attribute> list, Source source, FieldAttribute parent, Map<String, EsField> mapping) {
+    private static void mappingAsAttributes(List<Attribute> list, Source source, String parentName, Map<String, EsField> mapping) {
         for (Map.Entry<String, EsField> entry : mapping.entrySet()) {
             String name = entry.getKey();
             EsField t = entry.getValue();
 
             if (t != null) {
-                name = parent == null ? name : parent.fieldName() + "." + name;
+                name = parentName == null ? name : parentName + "." + name;
                 var fieldProperties = t.getProperties();
                 var type = t.getDataType().widenSmallNumeric();
                 // due to a bug also copy the field since the Attribute hierarchy extracts the data type
@@ -245,14 +245,14 @@ private static void mappingAsAttributes(List<Attribute> list, Source source, Fie
 
                 FieldAttribute attribute = t instanceof UnsupportedEsField uef
                     ? new UnsupportedAttribute(source, name, uef)
-                    : new FieldAttribute(source, parent, name, t);
+                    : new FieldAttribute(source, parentName, name, t);
                 // primitive branch
                 if (DataType.isPrimitive(type)) {
                     list.add(attribute);
                 }
                 // allow compound object even if they are unknown
                 if (fieldProperties.isEmpty() == false) {
-                    mappingAsAttributes(list, source, attribute, fieldProperties);
+                    mappingAsAttributes(list, source, attribute.name(), fieldProperties);
                 }
             }
         }
@@ -488,6 +488,7 @@ private LogicalPlan resolveStats(Stats stats, List<Attribute> childrenOutput) {
                     newAggregates.add(agg);
                 }
 
+                // TODO: remove this when Stats interface is removed
                 stats = changed.get() ? stats.with(stats.child(), groupings, newAggregates) : stats;
             }
 
@@ -1251,7 +1252,7 @@ private Expression createIfDoesNotAlreadyExist(
             // NOTE: The name has to start with $$ to not break bwc with 8.15 - in that version, this is how we had to mark this as
             // synthetic to work around a bug.
             String unionTypedFieldName = Attribute.rawTemporaryName(fa.name(), "converted_to", resolvedField.getDataType().typeName());
-            FieldAttribute unionFieldAttribute = new FieldAttribute(fa.source(), fa.parent(), unionTypedFieldName, resolvedField, true);
+            FieldAttribute unionFieldAttribute = new FieldAttribute(fa.source(), fa.parentName(), unionTypedFieldName, resolvedField, true);
             int existingIndex = unionFieldAttributes.indexOf(unionFieldAttribute);
             if (existingIndex >= 0) {
                 // Do not generate multiple name/type combinations with different IDs
@@ -1280,7 +1281,7 @@ private Expression typeSpecificConvert(AbstractConvertFunction convert, Source s
             FieldAttribute originalFieldAttr = (FieldAttribute) convert.field();
             FieldAttribute resolvedAttr = new FieldAttribute(
                 source,
-                originalFieldAttr.parent(),
+                originalFieldAttr.parentName(),
                 originalFieldAttr.name(),
                 field,
                 originalFieldAttr.nullable(),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
index 647a29b71c5e1..fbaf43467a2e7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
@@ -19,15 +19,22 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate;
 import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString;
 import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
@@ -51,10 +58,12 @@
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
 
@@ -183,7 +192,7 @@ else if (p instanceof Lookup lookup) {
 
             checkOperationsOnUnsignedLong(p, failures);
             checkBinaryComparison(p, failures);
-            checkForSortOnSpatialTypes(p, failures);
+            checkForSortableDataTypes(p, failures);
 
             checkFilterMatchConditions(p, failures);
             checkFullTextQueryFunctions(p, failures);
@@ -301,6 +310,29 @@ private static void checkInvalidNamedExpressionUsage(
         Set<Failure> failures,
         int level
     ) {
+        // unwrap filtered expression
+        if (e instanceof FilteredExpression fe) {
+            e = fe.delegate();
+            // make sure they work on aggregate functions
+            if (e.anyMatch(AggregateFunction.class::isInstance) == false) {
+                Expression filter = fe.filter();
+                failures.add(fail(filter, "WHERE clause allowed only for aggregate functions, none found in [{}]", fe.sourceText()));
+            }
+            // but that the filter doesn't use grouping or aggregate functions
+            fe.filter().forEachDown(c -> {
+                if (c instanceof AggregateFunction af) {
+                    failures.add(
+                        fail(af, "cannot use aggregate function [{}] in aggregate WHERE clause [{}]", af.sourceText(), fe.sourceText())
+                    );
+                }
+                // check the bucketing function against the group
+                else if (c instanceof GroupingFunction gf) {
+                    if (Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) {
+                        failures.add(fail(gf, "can only use grouping function [{}] part of the BY clause", gf.sourceText()));
+                    }
+                }
+            });
+        }
         // found an aggregate, constant or a group, bail out
         if (e instanceof AggregateFunction af) {
             af.field().forEachDown(AggregateFunction.class, f -> {
@@ -312,7 +344,7 @@ private static void checkInvalidNamedExpressionUsage(
         } else if (e instanceof GroupingFunction gf) {
             // optimizer will later unroll expressions with aggs and non-aggs with a grouping function into an EVAL, but that will no longer
             // be verified (by check above in checkAggregate()), so do it explicitly here
-            if (groups.stream().anyMatch(ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) {
+            if (Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) {
                 failures.add(fail(gf, "can only use grouping function [{}] part of the BY clause", gf.sourceText()));
             } else if (level == 0) {
                 addFailureOnGroupingUsedNakedInAggs(failures, gf, "function");
@@ -449,6 +481,9 @@ private void gatherMetrics(LogicalPlan plan, BitSet b) {
         for (int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) {
             metrics.inc(FeatureMetric.values()[i]);
         }
+        Set<Class<?>> functions = new HashSet<>();
+        plan.forEachExpressionDown(Function.class, p -> functions.add(p.getClass()));
+        functions.forEach(f -> metrics.incFunctionMetric(f));
     }
 
     /**
@@ -548,12 +583,12 @@ private static Failure validateUnsignedLongNegation(Neg neg) {
     }
 
     /**
-     * Makes sure that spatial types do not appear in sorting contexts.
+     * Some datatypes are not sortable
      */
-    private static void checkForSortOnSpatialTypes(LogicalPlan p, Set<Failure> localFailures) {
+    private static void checkForSortableDataTypes(LogicalPlan p, Set<Failure> localFailures) {
         if (p instanceof OrderBy ob) {
             ob.order().forEach(order -> {
-                if (DataType.isSpatial(order.dataType())) {
+                if (DataType.isSortable(order.dataType()) == false) {
                     localFailures.add(fail(order, "cannot sort on " + order.dataType().typeName()));
                 }
             });
@@ -574,22 +609,15 @@ private static void checkForSortOnSpatialTypes(LogicalPlan p, Set<Failure> local
      */
     private static void checkRemoteEnrich(LogicalPlan plan, Set<Failure> failures) {
         boolean[] agg = { false };
-        boolean[] limit = { false };
         boolean[] enrichCoord = { false };
 
         plan.forEachUp(UnaryPlan.class, u -> {
-            if (u instanceof Limit) {
-                limit[0] = true; // TODO: Make Limit then enrich_remote work
-            }
             if (u instanceof Aggregate) {
                 agg[0] = true;
             } else if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.COORDINATOR) {
                 enrichCoord[0] = true;
             }
             if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.REMOTE) {
-                if (limit[0]) {
-                    failures.add(fail(enrich, "ENRICH with remote policy can't be executed after LIMIT"));
-                }
                 if (agg[0]) {
                     failures.add(fail(enrich, "ENRICH with remote policy can't be executed after STATS"));
                 }
@@ -644,27 +672,105 @@ private static void checkFilterMatchConditions(LogicalPlan plan, Set<Failure> fa
     private static void checkFullTextQueryFunctions(LogicalPlan plan, Set<Failure> failures) {
         if (plan instanceof Filter f) {
             Expression condition = f.condition();
-            if (condition instanceof FullTextFunction ftf) {
-                // Similar to cases present in org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters -
-                // we can't check if it can be pushed down as we don't have yet information about the fields present in the
-                // StringQueryPredicate
-                plan.forEachDown(LogicalPlan.class, lp -> {
-                    if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) {
-                        failures.add(
-                            fail(
-                                plan,
-                                "[{}] function cannot be used after {}",
-                                ftf.functionName(),
-                                lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT)
-                            )
-                        );
-                    }
-                });
-            }
+            checkCommandsBeforeQueryStringFunction(plan, condition, failures);
+            checkCommandsBeforeMatchFunction(plan, condition, failures);
+            checkFullTextFunctionsConditions(condition, failures);
+            checkFullTextFunctionsParents(condition, failures);
         } else {
             plan.forEachExpression(FullTextFunction.class, ftf -> {
                 failures.add(fail(ftf, "[{}] function is only supported in WHERE commands", ftf.functionName()));
             });
         }
     }
+
+    private static void checkCommandsBeforeQueryStringFunction(LogicalPlan plan, Expression condition, Set<Failure> failures) {
+        condition.forEachDown(QueryString.class, qsf -> {
+            plan.forEachDown(LogicalPlan.class, lp -> {
+                if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) {
+                    failures.add(
+                        fail(
+                            plan,
+                            "[{}] function cannot be used after {}",
+                            qsf.functionName(),
+                            lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT)
+                        )
+                    );
+                }
+            });
+        });
+    }
+
+    private static void checkCommandsBeforeMatchFunction(LogicalPlan plan, Expression condition, Set<Failure> failures) {
+        condition.forEachDown(Match.class, qsf -> {
+            plan.forEachDown(LogicalPlan.class, lp -> {
+                if (lp instanceof Limit) {
+                    failures.add(
+                        fail(
+                            plan,
+                            "[{}] function cannot be used after {}",
+                            qsf.functionName(),
+                            lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT)
+                        )
+                    );
+                }
+            });
+        });
+    }
+
+    private static void checkFullTextFunctionsConditions(Expression condition, Set<Failure> failures) {
+        condition.forEachUp(Or.class, or -> {
+            checkFullTextFunctionInDisjunction(failures, or, or.left());
+            checkFullTextFunctionInDisjunction(failures, or, or.right());
+        });
+    }
+
+    private static void checkFullTextFunctionInDisjunction(Set<Failure> failures, Or or, Expression left) {
+        left.forEachDown(FullTextFunction.class, ftf -> {
+            failures.add(
+                fail(
+                    or,
+                    "Invalid condition [{}]. Function {} can't be used as part of an or condition",
+                    or.sourceText(),
+                    ftf.functionName()
+                )
+            );
+        });
+    }
+
+    private static void checkFullTextFunctionsParents(Expression condition, Set<Failure> failures) {
+        forEachFullTextFunctionParent(condition, (ftf, parent) -> {
+            if ((parent instanceof FullTextFunction == false)
+                && (parent instanceof BinaryLogic == false)
+                && (parent instanceof Not == false)) {
+                failures.add(
+                    fail(
+                        condition,
+                        "Invalid condition [{}]. Function {} can't be used with {}",
+                        condition.sourceText(),
+                        ftf.functionName(),
+                        ((Function) parent).functionName()
+                    )
+                );
+            }
+        });
+    }
+
+    /**
+     * Executes the action on every parent of a FullTextFunction in the condition if it is found
+     *
+     * @param action the action to execute for each parent of a FullTextFunction
+     */
+    private static FullTextFunction forEachFullTextFunctionParent(Expression condition, BiConsumer<FullTextFunction, Expression> action) {
+        if (condition instanceof FullTextFunction ftf) {
+            return ftf;
+        }
+        for (Expression child : condition.children()) {
+            FullTextFunction foundMatchingChild = forEachFullTextFunctionParent(child, action);
+            if (foundMatchingChild != null) {
+                action.accept(foundMatchingChild, condition);
+                return foundMatchingChild;
+            }
+        }
+        return null;
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
index 447df09942ca8..e67c406e26929 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportRequest;
@@ -72,12 +73,14 @@ public class EnrichPolicyResolver {
     private final IndexResolver indexResolver;
     private final TransportService transportService;
     private final ThreadPool threadPool;
+    private final RemoteClusterService remoteClusterService;
 
     public EnrichPolicyResolver(ClusterService clusterService, TransportService transportService, IndexResolver indexResolver) {
         this.clusterService = clusterService;
         this.transportService = transportService;
         this.indexResolver = indexResolver;
         this.threadPool = transportService.getThreadPool();
+        this.remoteClusterService = transportService.getRemoteClusterService();
         transportService.registerRequestHandler(
             RESOLVE_ACTION_NAME,
             threadPool.executor(ThreadPool.Names.SEARCH),
@@ -257,22 +260,21 @@ private void lookupPolicies(
             // remote clusters
             if (remotePolicies.isEmpty() == false) {
                 for (String cluster : remoteClusters) {
-                    final Transport.Connection connection;
-                    try {
-                        connection = getRemoteConnection(cluster);
-                    } catch (Exception e) {
-                        refs.acquire().onFailure(e);
-                        return;
-                    }
-                    transportService.sendRequest(
-                        connection,
-                        RESOLVE_ACTION_NAME,
-                        new LookupRequest(cluster, remotePolicies),
-                        TransportRequestOptions.EMPTY,
-                        new ActionListenerResponseHandler<>(
-                            refs.acquire(resp -> lookupResponses.put(cluster, resp)),
-                            LookupResponse::new,
-                            threadPool.executor(ThreadPool.Names.SEARCH)
+                    ActionListener<LookupResponse> lookupListener = refs.acquire(resp -> lookupResponses.put(cluster, resp));
+                    getRemoteConnection(
+                        cluster,
+                        lookupListener.delegateFailureAndWrap(
+                            (delegate, connection) -> transportService.sendRequest(
+                                connection,
+                                RESOLVE_ACTION_NAME,
+                                new LookupRequest(cluster, remotePolicies),
+                                TransportRequestOptions.EMPTY,
+                                new ActionListenerResponseHandler<>(
+                                    delegate,
+                                    LookupResponse::new,
+                                    threadPool.executor(ThreadPool.Names.SEARCH)
+                                )
+                            )
                         )
                     );
                 }
@@ -389,13 +391,16 @@ protected Map<String, EnrichPolicy> availablePolicies() {
         return metadata == null ? Map.of() : metadata.getPolicies();
     }
 
-    protected Transport.Connection getRemoteConnection(String cluster) {
-        return transportService.getRemoteClusterService().getConnection(cluster);
+    protected void getRemoteConnection(String cluster, ActionListener<Transport.Connection> listener) {
+        remoteClusterService.maybeEnsureConnectedAndGetConnection(
+            cluster,
+            remoteClusterService.isSkipUnavailable(cluster) == false,
+            listener
+        );
     }
 
     public Map<String, List<String>> groupIndicesPerCluster(String[] indices) {
-        return transportService.getRemoteClusterService()
-            .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, indices)
+        return remoteClusterService.groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, indices)
             .entrySet()
             .stream()
             .collect(Collectors.toMap(Map.Entry::getKey, e -> Arrays.asList(e.getValue().indices())));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperator.java
index 6937f1a8c7772..f70cfe1dc8a41 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperator.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperator.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Query;
@@ -89,7 +90,7 @@ public Page getOutput() {
                         continue;
                     }
                     final DocCollector collector = new DocCollector(docsBuilder);
-                    scorer.score(collector, leaf.reader().getLiveDocs());
+                    scorer.score(collector, leaf.reader().getLiveDocs(), 0, DocIdSetIterator.NO_MORE_DOCS);
                     int matches = collector.matches;
 
                     if (segmentsBuilder != null) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java
index 1ee9039ccccb3..e891089aa55b5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java
@@ -36,7 +36,7 @@ public ResolvedEnrichPolicy(StreamInput in) throws IOException {
 
     private static Reader<EsField> getEsFieldReader(StreamInput in) {
         if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)
-            || in.getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)) {
+            || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) {
             return EsField::readFrom;
         }
         return EsField::new;
@@ -57,7 +57,7 @@ public void writeTo(StreamOutput out) throws IOException {
             (o, v) -> {
                 var field = new EsField(v.getName(), v.getDataType(), v.getProperties(), v.isAggregatable(), v.isAlias());
                 if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)
-                    || out.getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)) {
+                    || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) {
                     field.writeTo(o);
                 } else {
                     field.writeContent(o);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
index 7d8e0cd736445..ee8822889bedb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
@@ -48,7 +48,7 @@ public PlanExecutor(IndexResolver indexResolver, MeterRegistry meterRegistry) {
         this.preAnalyzer = new PreAnalyzer();
         this.functionRegistry = new EsqlFunctionRegistry();
         this.mapper = new Mapper(functionRegistry);
-        this.metrics = new Metrics();
+        this.metrics = new Metrics(functionRegistry);
         this.verifier = new Verifier(metrics);
         this.planningMetricsManager = new PlanningMetricsManager(meterRegistry);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index 8e238f9ed760c..66151275fc2e8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -32,7 +32,8 @@
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Top;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Values;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryStringFunction;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString;
 import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket;
 import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize;
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
@@ -78,6 +79,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.E;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Exp;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Floor;
+import org.elasticsearch.xpack.esql.expression.function.scalar.math.Hypot;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log10;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pi;
@@ -257,19 +259,21 @@ private FunctionDefinition[][] functions() {
             // grouping functions
             new FunctionDefinition[] { def(Bucket.class, Bucket::new, "bucket", "bin"), },
             // aggregate functions
+            // since they declare two public constructors - one with filter (for nested where) and one without
+            // use casting to disambiguate between the two
             new FunctionDefinition[] {
-                def(Avg.class, Avg::new, "avg"),
-                def(Count.class, Count::new, "count"),
-                def(CountDistinct.class, CountDistinct::new, "count_distinct"),
-                def(Max.class, Max::new, "max"),
-                def(Median.class, Median::new, "median"),
-                def(MedianAbsoluteDeviation.class, MedianAbsoluteDeviation::new, "median_absolute_deviation"),
-                def(Min.class, Min::new, "min"),
-                def(Percentile.class, Percentile::new, "percentile"),
-                def(Sum.class, Sum::new, "sum"),
-                def(Top.class, Top::new, "top"),
-                def(Values.class, Values::new, "values"),
-                def(WeightedAvg.class, WeightedAvg::new, "weighted_avg") },
+                def(Avg.class, uni(Avg::new), "avg"),
+                def(Count.class, uni(Count::new), "count"),
+                def(CountDistinct.class, bi(CountDistinct::new), "count_distinct"),
+                def(Max.class, uni(Max::new), "max"),
+                def(Median.class, uni(Median::new), "median"),
+                def(MedianAbsoluteDeviation.class, uni(MedianAbsoluteDeviation::new), "median_absolute_deviation"),
+                def(Min.class, uni(Min::new), "min"),
+                def(Percentile.class, bi(Percentile::new), "percentile"),
+                def(Sum.class, uni(Sum::new), "sum"),
+                def(Top.class, tri(Top::new), "top"),
+                def(Values.class, uni(Values::new), "values"),
+                def(WeightedAvg.class, bi(WeightedAvg::new), "weighted_avg") },
             // math
             new FunctionDefinition[] {
                 def(Abs.class, Abs::new, "abs"),
@@ -285,6 +289,7 @@ private FunctionDefinition[][] functions() {
                 def(Exp.class, Exp::new, "exp"),
                 def(Floor.class, Floor::new, "floor"),
                 def(Greatest.class, Greatest::new, "greatest"),
+                def(Hypot.class, Hypot::new, "hypot"),
                 def(Log.class, Log::new, "log"),
                 def(Log10.class, Log10::new, "log10"),
                 def(Least.class, Least::new, "least"),
@@ -383,7 +388,9 @@ private FunctionDefinition[][] functions() {
                 def(MvSlice.class, MvSlice::new, "mv_slice"),
                 def(MvZip.class, MvZip::new, "mv_zip"),
                 def(MvSum.class, MvSum::new, "mv_sum"),
-                def(Split.class, Split::new, "split") } };
+                def(Split.class, Split::new, "split") },
+            // fulltext functions
+            new FunctionDefinition[] { def(Match.class, Match::new, "match"), def(QueryString.class, QueryString::new, "qstr") } };
 
     }
 
@@ -391,9 +398,7 @@ private static FunctionDefinition[][] snapshotFunctions() {
         return new FunctionDefinition[][] {
             new FunctionDefinition[] {
                 def(Categorize.class, Categorize::new, "categorize"),
-                def(Rate.class, Rate::withUnresolvedTimestamp, "rate"),
-                // Full text functions
-                def(QueryStringFunction.class, QueryStringFunction::new, "qstr") } };
+                def(Rate.class, Rate::withUnresolvedTimestamp, "rate") } };
     }
 
     public EsqlFunctionRegistry snapshotRegistry() {
@@ -479,11 +484,10 @@ public static DataType getTargetType(String[] names) {
     }
 
     public static FunctionDescription description(FunctionDefinition def) {
-        var constructors = def.clazz().getConstructors();
-        if (constructors.length == 0) {
+        Constructor<?> constructor = constructorFor(def.clazz());
+        if (constructor == null) {
             return new FunctionDescription(def.name(), List.of(), null, null, false, false);
         }
-        Constructor<?> constructor = constructors[0];
         FunctionInfo functionInfo = functionInfo(def);
         String functionDescription = functionInfo == null ? "" : functionInfo.description().replace('\n', ' ');
         String[] returnType = functionInfo == null ? new String[] { "?" } : removeUnderConstruction(functionInfo.returnType());
@@ -520,14 +524,29 @@ private static String[] removeUnderConstruction(String[] types) {
     }
 
     public static FunctionInfo functionInfo(FunctionDefinition def) {
-        var constructors = def.clazz().getConstructors();
-        if (constructors.length == 0) {
+        Constructor<?> constructor = constructorFor(def.clazz());
+        if (constructor == null) {
             return null;
         }
-        Constructor<?> constructor = constructors[0];
         return constructor.getAnnotation(FunctionInfo.class);
     }
 
+    private static Constructor<?> constructorFor(Class<? extends Function> clazz) {
+        Constructor<?>[] constructors = clazz.getConstructors();
+        if (constructors.length == 0) {
+            return null;
+        }
+        // when dealing with multiple, pick the constructor exposing the FunctionInfo annotation
+        if (constructors.length > 1) {
+            for (Constructor<?> constructor : constructors) {
+                if (constructor.getAnnotation(FunctionInfo.class) != null) {
+                    return constructor;
+                }
+            }
+        }
+        return constructors[0];
+    }
+
     private void buildDataTypesForStringLiteralConversion(FunctionDefinition[]... groupFunctions) {
         for (FunctionDefinition[] group : groupFunctions) {
             for (FunctionDefinition def : group) {
@@ -910,15 +929,19 @@ protected interface TernaryConfigurationAwareBuilder<T> {
     }
 
     //
-    // Utility method for extra argument extraction.
+    // Utility functions to help disambiguate the method handle passed in.
+    // They work by providing additional method information to help the compiler know which method to pick.
     //
-    protected static Boolean asBool(Object[] extras) {
-        if (CollectionUtils.isEmpty(extras)) {
-            return null;
-        }
-        if (extras.length != 1 || (extras[0] instanceof Boolean) == false) {
-            throw new QlIllegalArgumentException("Invalid number and types of arguments given to function definition");
-        }
-        return (Boolean) extras[0];
+    private static <T extends Function> BiFunction<Source, Expression, T> uni(BiFunction<Source, Expression, T> function) {
+        return function;
     }
+
+    private static <T extends Function> BinaryBuilder<T> bi(BinaryBuilder<T> function) {
+        return function;
+    }
+
+    private static <T extends Function> TernaryBuilder<T> tri(TernaryBuilder<T> function) {
+        return function;
+    }
+
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
index 08c249662c7d2..d372eddb961ae 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -29,6 +30,9 @@
 import java.io.IOException;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * Unsupported attribute meaning an attribute that has been found yet cannot be used (hence why UnresolvedAttribute
  * cannot be used) expect in special conditions (currently only in projections to allow it to flow through
@@ -63,11 +67,11 @@ public UnsupportedAttribute(Source source, String name, UnsupportedEsField field
         this(source, name, field, null);
     }
 
-    public UnsupportedAttribute(Source source, String name, UnsupportedEsField field, String customMessage) {
+    public UnsupportedAttribute(Source source, String name, UnsupportedEsField field, @Nullable String customMessage) {
         this(source, name, field, customMessage, null);
     }
 
-    public UnsupportedAttribute(Source source, String name, UnsupportedEsField field, String customMessage, NameId id) {
+    public UnsupportedAttribute(Source source, String name, UnsupportedEsField field, @Nullable String customMessage, @Nullable NameId id) {
         super(source, null, name, field, Nullability.TRUE, id, false);
         this.hasCustomMessage = customMessage != null;
         this.message = customMessage == null ? errorMessage(name(), field) : customMessage;
@@ -76,11 +80,9 @@ public UnsupportedAttribute(Source source, String name, UnsupportedEsField field
     private UnsupportedAttribute(StreamInput in) throws IOException {
         this(
             Source.readFrom((PlanStreamInput) in),
-            ((PlanStreamInput) in).readCachedString(),
+            readCachedStringWithVersionCheck(in),
             in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)
-                || in.getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)
-                    ? EsField.readFrom(in)
-                    : new UnsupportedEsField(in),
+                || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2) ? EsField.readFrom(in) : new UnsupportedEsField(in),
             in.readOptionalString(),
             NameId.readFrom((PlanStreamInput) in)
         );
@@ -90,9 +92,9 @@ private UnsupportedAttribute(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) {
             Source.EMPTY.writeTo(out);
-            ((PlanStreamOutput) out).writeCachedString(name());
+            writeCachedStringWithVersionCheck(out, name());
             if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)
-                || out.getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)) {
+                || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) {
                 field().writeTo(out);
             } else {
                 field().writeContent(out);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java
index f0acac0e9744e..f7a74cc2ae93f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java
@@ -6,10 +6,12 @@
  */
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -20,8 +22,8 @@
 import java.util.List;
 import java.util.Objects;
 
+import static java.util.Arrays.asList;
 import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 
 /**
@@ -52,25 +54,51 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
 
     private final Expression field;
     private final List<? extends Expression> parameters;
+    private final Expression filter;
 
     protected AggregateFunction(Source source, Expression field) {
-        this(source, field, emptyList());
+        this(source, field, Literal.TRUE, emptyList());
     }
 
     protected AggregateFunction(Source source, Expression field, List<? extends Expression> parameters) {
-        super(source, CollectionUtils.combine(singletonList(field), parameters));
+        this(source, field, Literal.TRUE, parameters);
+    }
+
+    protected AggregateFunction(Source source, Expression field, Expression filter, List<? extends Expression> parameters) {
+        super(source, CollectionUtils.combine(asList(field, filter), parameters));
         this.field = field;
+        this.filter = filter;
         this.parameters = parameters;
     }
 
     protected AggregateFunction(StreamInput in) throws IOException {
-        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class));
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(Expression.class),
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class)
+                : emptyList()
+        );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public final void writeTo(StreamOutput out) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeNamedWriteable(field);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)) {
+            out.writeNamedWriteable(filter);
+            out.writeNamedWriteableCollection(parameters);
+        } else {
+            deprecatedWriteParams(out);
+        }
+    }
+
+    @Deprecated(since = "8.16", forRemoval = true)
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
+        //
     }
 
     public Expression field() {
@@ -81,12 +109,12 @@ public List<? extends Expression> parameters() {
         return parameters;
     }
 
-    /**
-     * Returns the input expressions used in aggregation.
-     * Defaults to a list containing the only the input field.
-     */
-    public List<Expression> inputExpressions() {
-        return List.of(field);
+    public boolean hasFilter() {
+        return filter != null && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold()) == false);
+    }
+
+    public Expression filter() {
+        return filter;
     }
 
     @Override
@@ -94,6 +122,18 @@ protected TypeResolution resolveType() {
         return TypeResolutions.isExact(field, sourceText(), DEFAULT);
     }
 
+    /**
+     * Attach a filter to the aggregate function.
+     */
+    public abstract AggregateFunction withFilter(Expression filter);
+
+    public AggregateFunction withParameters(List<? extends Expression> parameters) {
+        if (parameters == this.parameters) {
+            return this;
+        }
+        return (AggregateFunction) replaceChildren(CollectionUtils.combine(asList(field, filter), parameters));
+    }
+
     @Override
     public int hashCode() {
         // NB: the hashcode is currently used for key generation so
@@ -105,7 +145,9 @@ public int hashCode() {
     public boolean equals(Object obj) {
         if (super.equals(obj)) {
             AggregateFunction other = (AggregateFunction) obj;
-            return Objects.equals(other.field(), field()) && Objects.equals(other.parameters(), parameters());
+            return Objects.equals(other.field(), field())
+                && Objects.equals(other.filter(), filter())
+                && Objects.equals(other.parameters(), parameters());
         }
         return false;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java
index b5c0b8e5ffdc8..82c0f9d24899e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -23,6 +24,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 
@@ -44,7 +46,11 @@ public class Avg extends AggregateFunction implements SurrogateExpression {
             ) }
     )
     public Avg(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) {
-        super(source, field);
+        this(source, field, Literal.TRUE);
+    }
+
+    public Avg(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     @Override
@@ -74,12 +80,17 @@ public DataType dataType() {
 
     @Override
     protected NodeInfo<Avg> info() {
-        return NodeInfo.create(this, Avg::new, field());
+        return NodeInfo.create(this, Avg::new, field(), filter());
     }
 
     @Override
     public Avg replaceChildren(List<Expression> newChildren) {
-        return new Avg(source(), newChildren.get(0));
+        return new Avg(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public Avg withFilter(Expression filter) {
+        return new Avg(source(), field(), filter);
     }
 
     @Override
@@ -87,6 +98,8 @@ public Expression surrogate() {
         var s = source();
         var field = field();
 
-        return field().foldable() ? new MvAvg(s, field) : new Div(s, new Sum(s, field), new Count(s, field), dataType());
+        return field().foldable()
+            ? new MvAvg(s, field)
+            : new Div(s, new Sum(s, field, filter()), new Count(s, field, filter()), dataType());
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
index 9b6190408dbd4..fa8a9e7d8c837 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
@@ -30,10 +30,11 @@
 import java.io.IOException;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 
-public class Count extends AggregateFunction implements EnclosedAgg, ToAggregator, SurrogateExpression {
+public class Count extends AggregateFunction implements ToAggregator, SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Count", Count::new);
 
     @FunctionInfo(
@@ -83,7 +84,11 @@ public Count(
             description = "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
         ) Expression field
     ) {
-        super(source, field);
+        this(source, field, Literal.TRUE);
+    }
+
+    public Count(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     private Count(StreamInput in) throws IOException {
@@ -97,17 +102,17 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<Count> info() {
-        return NodeInfo.create(this, Count::new, field());
+        return NodeInfo.create(this, Count::new, field(), filter());
     }
 
     @Override
-    public Count replaceChildren(List<Expression> newChildren) {
-        return new Count(source(), newChildren.get(0));
+    public AggregateFunction withFilter(Expression filter) {
+        return new Count(source(), field(), filter);
     }
 
     @Override
-    public String innerName() {
-        return "count";
+    public Count replaceChildren(List<Expression> newChildren) {
+        return new Count(source(), newChildren.get(0), newChildren.get(1));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
index 858c6e659449c..5ae162f1fbb12 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -37,12 +38,15 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isWholeNumber;
+import static org.elasticsearch.xpack.esql.core.util.CollectionUtils.nullSafeList;
 
 public class CountDistinct extends AggregateFunction implements OptionalArgument, ToAggregator, SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
@@ -51,6 +55,20 @@ public class CountDistinct extends AggregateFunction implements OptionalArgument
         CountDistinct::new
     );
 
+    private static final Map<DataType, BiFunction<List<Integer>, Integer, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries(
+        // Booleans ignore the precision because there are only two possible values anyway
+        Map.entry(DataType.BOOLEAN, (inputChannels, precision) -> new CountDistinctBooleanAggregatorFunctionSupplier(inputChannels)),
+        Map.entry(DataType.LONG, CountDistinctLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATETIME, CountDistinctLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATE_NANOS, CountDistinctLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.INTEGER, CountDistinctIntAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DOUBLE, CountDistinctDoubleAggregatorFunctionSupplier::new),
+        Map.entry(DataType.KEYWORD, CountDistinctBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.IP, CountDistinctBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.VERSION, CountDistinctBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new)
+    );
+
     private static final int DEFAULT_PRECISION = 3000;
     private final Expression precision;
 
@@ -100,7 +118,7 @@ public CountDistinct(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
             description = "Column or literal for which to count the number of distinct values."
         ) Expression field,
         @Param(
@@ -112,22 +130,33 @@ public CountDistinct(
                 + "same effect as a threshold of 40000. The default value is 3000."
         ) Expression precision
     ) {
-        super(source, field, precision != null ? List.of(precision) : List.of());
-        this.precision = precision;
+        this(source, field, Literal.TRUE, precision);
+    }
+
+    public CountDistinct(Source source, Expression field, Expression filter, Expression precision) {
+        this(source, field, filter, precision != null ? List.of(precision) : List.of());
+    }
+
+    private CountDistinct(Source source, Expression field, Expression filter, List<Expression> params) {
+        super(source, field, filter, params);
+        this.precision = params.size() > 0 ? params.get(0) : null;
     }
 
     private CountDistinct(StreamInput in) throws IOException {
         this(
             Source.readFrom((PlanStreamInput) in),
             in.readNamedWriteable(Expression.class),
-            in.readOptionalNamedWriteable(Expression.class)
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class)
+                : nullSafeList(in.readOptionalNamedWriteable(Expression.class))
         );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeNamedWriteable(field());
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
         out.writeOptionalNamedWriteable(precision);
     }
 
@@ -138,12 +167,17 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<CountDistinct> info() {
-        return NodeInfo.create(this, CountDistinct::new, field(), precision);
+        return NodeInfo.create(this, CountDistinct::new, field(), filter(), precision);
     }
 
     @Override
     public CountDistinct replaceChildren(List<Expression> newChildren) {
-        return new CountDistinct(source(), newChildren.get(0), newChildren.size() > 1 ? newChildren.get(1) : null);
+        return new CountDistinct(source(), newChildren.get(0), newChildren.get(1), newChildren.size() > 2 ? newChildren.get(2) : null);
+    }
+
+    @Override
+    public CountDistinct withFilter(Expression filter) {
+        return new CountDistinct(source(), field(), filter, precision);
     }
 
     @Override
@@ -161,7 +195,7 @@ protected TypeResolution resolveType() {
             .and(
                 isType(
                     field(),
-                    dt -> dt != DataType.UNSIGNED_LONG && dt != DataType.SOURCE,
+                    SUPPLIERS::containsKey,
                     sourceText(),
                     DEFAULT,
                     "any exact type except unsigned_long, _source, or counter types"
@@ -178,23 +212,11 @@ protected TypeResolution resolveType() {
     public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
         int precision = this.precision == null ? DEFAULT_PRECISION : ((Number) this.precision.fold()).intValue();
-        if (type == DataType.BOOLEAN) {
-            // Booleans ignore the precision because there are only two possible values anyway
-            return new CountDistinctBooleanAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.DATETIME || type == DataType.LONG) {
-            return new CountDistinctLongAggregatorFunctionSupplier(inputChannels, precision);
-        }
-        if (type == DataType.INTEGER) {
-            return new CountDistinctIntAggregatorFunctionSupplier(inputChannels, precision);
-        }
-        if (type == DataType.DOUBLE) {
-            return new CountDistinctDoubleAggregatorFunctionSupplier(inputChannels, precision);
-        }
-        if (type == DataType.KEYWORD || type == DataType.IP || type == DataType.VERSION || type == DataType.TEXT) {
-            return new CountDistinctBytesRefAggregatorFunctionSupplier(inputChannels, precision);
+        if (SUPPLIERS.containsKey(type) == false) {
+            // If the type checking did its job, this should never happen
+            throw EsqlIllegalArgumentException.illegalDataType(type);
         }
-        throw EsqlIllegalArgumentException.illegalDataType(type);
+        return SUPPLIERS.get(type).apply(inputChannels, precision);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FilteredExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FilteredExpression.java
new file mode 100644
index 0000000000000..97c6fb6dbd887
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FilteredExpression.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+
+/**
+ * Basic wrapper for expressions declared with a nested filter (typically in stats).
+ * Used during parsing to attach the filter to the nested expression - it is expected the two
+ * get fused later on.
+ */
+// TODO: This class should implement SurrogateExpression but it doesn't due to its use on folding aggregates
+// see https://github.com/elastic/elasticsearch/issues/100634#issuecomment-2400665066
+public class FilteredExpression extends Expression {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Expression.class,
+        "FilteredExpression",
+        FilteredExpression::new
+    );
+
+    private final Expression delegate;
+    private final Expression filter;
+
+    public FilteredExpression(Source source, Expression delegate, Expression filter) {
+        super(source, asList(delegate, filter));
+        this.delegate = delegate;
+        this.filter = filter;
+    }
+
+    public FilteredExpression(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+    }
+
+    public Expression surrogate() {
+        return delegate.transformUp(AggregateFunction.class, af -> af.withFilter(filter));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(delegate);
+        out.writeNamedWriteable(filter);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    public Expression delegate() {
+        return delegate;
+    }
+
+    public Expression filter() {
+        return filter;
+    }
+
+    @Override
+    public DataType dataType() {
+        return delegate.dataType();
+    }
+
+    @Override
+    public Nullability nullable() {
+        return delegate.nullable();
+    }
+
+    @Override
+    protected NodeInfo<FilteredExpression> info() {
+        return NodeInfo.create(this, FilteredExpression::new, delegate, filter);
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new FilteredExpression(source(), newChildren.get(0), newChildren.get(1));
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java
index 593e6fa463371..0f9037a28d7d7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -21,6 +22,7 @@
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -44,18 +46,29 @@ public class FromPartial extends AggregateFunction implements ToAggregator {
     private final Expression function;
 
     public FromPartial(Source source, Expression field, Expression function) {
-        super(source, field, List.of(function));
+        this(source, field, Literal.TRUE, function);
+    }
+
+    public FromPartial(Source source, Expression field, Expression filter, Expression function) {
+        super(source, field, filter, List.of(function));
         this.function = function;
     }
 
     private FromPartial(StreamInput in) throws IOException {
-        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(Expression.class),
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class).get(0)
+                : in.readNamedWriteable(Expression.class)
+        );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        source().writeTo(out);
-        out.writeNamedWriteable(field());
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
         out.writeNamedWriteable(function);
     }
 
@@ -85,12 +98,17 @@ public AttributeSet references() {
 
     @Override
     public Expression replaceChildren(List<Expression> newChildren) {
-        return new FromPartial(source(), newChildren.get(0), newChildren.get(1));
+        return new FromPartial(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
     }
 
     @Override
     protected NodeInfo<? extends Expression> info() {
-        return NodeInfo.create(this, FromPartial::new, field(), function);
+        return NodeInfo.create(this, FromPartial::new, field(), filter(), function);
+    }
+
+    @Override
+    public FromPartial withFilter(Expression filter) {
+        return new FromPartial(source(), field(), filter, function);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
index e7f790f90803a..ee16193efdccc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -31,15 +32,28 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial;
 
 public class Max extends AggregateFunction implements ToAggregator, SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Max", Max::new);
 
+    private static final Map<DataType, Function<List<Integer>, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries(
+        Map.entry(DataType.BOOLEAN, MaxBooleanAggregatorFunctionSupplier::new),
+        Map.entry(DataType.LONG, MaxLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATETIME, MaxLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATE_NANOS, MaxLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.INTEGER, MaxIntAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DOUBLE, MaxDoubleAggregatorFunctionSupplier::new),
+        Map.entry(DataType.IP, MaxIpAggregatorFunctionSupplier::new),
+        Map.entry(DataType.KEYWORD, MaxBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.TEXT, MaxBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.VERSION, MaxBytesRefAggregatorFunctionSupplier::new)
+    );
+
     @FunctionInfo(
         returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" },
         description = "The maximum value of a field.",
@@ -61,7 +75,11 @@ public Max(
             type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }
         ) Expression field
     ) {
-        super(source, field);
+        this(source, field, Literal.TRUE);
+    }
+
+    public Max(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     private Max(StreamInput in) throws IOException {
@@ -73,21 +91,26 @@ public String getWriteableName() {
         return ENTRY.name;
     }
 
+    @Override
+    public Max withFilter(Expression filter) {
+        return new Max(source(), field(), filter);
+    }
+
     @Override
     protected NodeInfo<Max> info() {
-        return NodeInfo.create(this, Max::new, field());
+        return NodeInfo.create(this, Max::new, field(), filter());
     }
 
     @Override
     public Max replaceChildren(List<Expression> newChildren) {
-        return new Max(source(), newChildren.get(0));
+        return new Max(source(), newChildren.get(0), newChildren.get(1));
     }
 
     @Override
     protected TypeResolution resolveType() {
         return TypeResolutions.isType(
             field(),
-            t -> isRepresentable(t) && t != UNSIGNED_LONG && isSpatial(t) == false,
+            SUPPLIERS::containsKey,
             sourceText(),
             DEFAULT,
             "representable except unsigned_long and spatial types"
@@ -102,25 +125,11 @@ public DataType dataType() {
     @Override
     public final AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
-        if (type == DataType.BOOLEAN) {
-            return new MaxBooleanAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.LONG || type == DataType.DATETIME) {
-            return new MaxLongAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.INTEGER) {
-            return new MaxIntAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.DOUBLE) {
-            return new MaxDoubleAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.IP) {
-            return new MaxIpAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) {
-            return new MaxBytesRefAggregatorFunctionSupplier(inputChannels);
+        if (SUPPLIERS.containsKey(type) == false) {
+            // If the type checking did its job, this should never happen
+            throw EsqlIllegalArgumentException.illegalDataType(type);
         }
-        throw EsqlIllegalArgumentException.illegalDataType(type);
+        return SUPPLIERS.get(type).apply(inputChannels);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
index 348fef577c934..c47fa612c1c49 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 
@@ -55,7 +56,11 @@ public class Median extends AggregateFunction implements SurrogateExpression {
             ), }
     )
     public Median(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) {
-        super(source, field);
+        this(source, field, Literal.TRUE);
+    }
+
+    public Median(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     @Override
@@ -85,12 +90,17 @@ public DataType dataType() {
 
     @Override
     protected NodeInfo<Median> info() {
-        return NodeInfo.create(this, Median::new, field());
+        return NodeInfo.create(this, Median::new, field(), filter());
     }
 
     @Override
     public Median replaceChildren(List<Expression> newChildren) {
-        return new Median(source(), newChildren.get(0));
+        return new Median(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public AggregateFunction withFilter(Expression filter) {
+        return new Median(source(), field(), filter);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java
index 23a6b23a35cde..dfcbd6d22abae 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.compute.aggregation.MedianAbsoluteDeviationIntAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MedianAbsoluteDeviationLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
@@ -26,6 +27,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
+
 public class MedianAbsoluteDeviation extends NumericAggregate implements SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Expression.class,
@@ -64,7 +67,11 @@ public class MedianAbsoluteDeviation extends NumericAggregate implements Surroga
             ), }
     )
     public MedianAbsoluteDeviation(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) {
-        super(source, field);
+        this(source, field, Literal.TRUE);
+    }
+
+    public MedianAbsoluteDeviation(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     private MedianAbsoluteDeviation(StreamInput in) throws IOException {
@@ -78,12 +85,17 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<MedianAbsoluteDeviation> info() {
-        return NodeInfo.create(this, MedianAbsoluteDeviation::new, field());
+        return NodeInfo.create(this, MedianAbsoluteDeviation::new, field(), filter());
     }
 
     @Override
     public MedianAbsoluteDeviation replaceChildren(List<Expression> newChildren) {
-        return new MedianAbsoluteDeviation(source(), newChildren.get(0));
+        return new MedianAbsoluteDeviation(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public MedianAbsoluteDeviation withFilter(Expression filter) {
+        return new MedianAbsoluteDeviation(source(), field(), filter);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
index 6866811995059..7aaa41ea6ab11 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.compute.aggregation.MinLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -31,15 +32,28 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial;
 
 public class Min extends AggregateFunction implements ToAggregator, SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Min", Min::new);
 
+    private static final Map<DataType, Function<List<Integer>, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries(
+        Map.entry(DataType.BOOLEAN, MinBooleanAggregatorFunctionSupplier::new),
+        Map.entry(DataType.LONG, MinLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATETIME, MinLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATE_NANOS, MinLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.INTEGER, MinIntAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DOUBLE, MinDoubleAggregatorFunctionSupplier::new),
+        Map.entry(DataType.IP, MinIpAggregatorFunctionSupplier::new),
+        Map.entry(DataType.VERSION, MinBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.KEYWORD, MinBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.TEXT, MinBytesRefAggregatorFunctionSupplier::new)
+    );
+
     @FunctionInfo(
         returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" },
         description = "The minimum value of a field.",
@@ -61,7 +75,11 @@ public Min(
             type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }
         ) Expression field
     ) {
-        super(source, field);
+        this(source, field, Literal.TRUE);
+    }
+
+    public Min(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     private Min(StreamInput in) throws IOException {
@@ -75,19 +93,24 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<Min> info() {
-        return NodeInfo.create(this, Min::new, field());
+        return NodeInfo.create(this, Min::new, field(), filter());
     }
 
     @Override
     public Min replaceChildren(List<Expression> newChildren) {
-        return new Min(source(), newChildren.get(0));
+        return new Min(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public Min withFilter(Expression filter) {
+        return new Min(source(), field(), filter);
     }
 
     @Override
     protected TypeResolution resolveType() {
         return TypeResolutions.isType(
             field(),
-            t -> isRepresentable(t) && t != UNSIGNED_LONG && isSpatial(t) == false,
+            SUPPLIERS::containsKey,
             sourceText(),
             DEFAULT,
             "representable except unsigned_long and spatial types"
@@ -102,25 +125,11 @@ public DataType dataType() {
     @Override
     public final AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
-        if (type == DataType.BOOLEAN) {
-            return new MinBooleanAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.LONG || type == DataType.DATETIME) {
-            return new MinLongAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.INTEGER) {
-            return new MinIntAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.DOUBLE) {
-            return new MinDoubleAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.IP) {
-            return new MinIpAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) {
-            return new MinBytesRefAggregatorFunctionSupplier(inputChannels);
+        if (SUPPLIERS.containsKey(type) == false) {
+            // If the type checking did its job, this should never happen
+            throw EsqlIllegalArgumentException.illegalDataType(type);
         }
-        throw EsqlIllegalArgumentException.illegalDataType(type);
+        return SUPPLIERS.get(type).apply(inputChannels);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java
index e7825a1d11704..5c639c465c649 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java
@@ -49,6 +49,10 @@ public abstract class NumericAggregate extends AggregateFunction implements ToAg
         super(source, field, parameters);
     }
 
+    NumericAggregate(Source source, Expression field, Expression filter, List<Expression> parameters) {
+        super(source, field, filter, parameters);
+    }
+
     NumericAggregate(Source source, Expression field) {
         super(source, field);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
index 0d5dd4b66501c..febd9f28b2291 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -15,6 +16,7 @@
 import org.elasticsearch.compute.aggregation.PercentileIntAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.PercentileLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -29,6 +31,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable;
@@ -77,19 +80,30 @@ public Percentile(
         @Param(name = "number", type = { "double", "integer", "long" }) Expression field,
         @Param(name = "percentile", type = { "double", "integer", "long" }) Expression percentile
     ) {
-        super(source, field, List.of(percentile));
+        this(source, field, Literal.TRUE, percentile);
+    }
+
+    public Percentile(Source source, Expression field, Expression filter, Expression percentile) {
+        super(source, field, filter, singletonList(percentile));
         this.percentile = percentile;
     }
 
     private Percentile(StreamInput in) throws IOException {
-        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(Expression.class),
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class).get(0)
+                : in.readNamedWriteable(Expression.class)
+        );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeNamedWriteable(children().get(0));
-        out.writeNamedWriteable(children().get(1));
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(percentile);
     }
 
     @Override
@@ -99,12 +113,17 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<Percentile> info() {
-        return NodeInfo.create(this, Percentile::new, field(), percentile);
+        return NodeInfo.create(this, Percentile::new, field(), filter(), percentile);
     }
 
     @Override
     public Percentile replaceChildren(List<Expression> newChildren) {
-        return new Percentile(source(), newChildren.get(0), newChildren.get(1));
+        return new Percentile(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+
+    @Override
+    public Percentile withFilter(Expression filter) {
+        return new Percentile(source(), field(), filter, percentile);
     }
 
     public Expression percentile() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
index f5597b7d64e81..b7b04658f8d58 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -17,6 +18,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -34,6 +36,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.util.CollectionUtils.nullSafeList;
 
 public class Rate extends AggregateFunction implements OptionalArgument, ToAggregator {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Rate", Rate::new);
@@ -53,7 +56,16 @@ public Rate(
         Expression timestamp,
         @Param(optional = true, name = "unit", type = { "time_duration" }, description = "the unit") Expression unit
     ) {
-        super(source, field, unit != null ? List.of(timestamp, unit) : List.of(timestamp));
+        this(source, field, Literal.TRUE, timestamp, unit);
+    }
+
+    // compatibility constructor used when reading from the stream
+    private Rate(Source source, Expression field, Expression filter, List<Expression> children) {
+        this(source, field, filter, children.get(0), children.size() > 1 ? children.get(1) : null);
+    }
+
+    private Rate(Source source, Expression field, Expression filter, Expression timestamp, Expression unit) {
+        super(source, field, filter, unit != null ? List.of(timestamp, unit) : List.of(timestamp));
         this.timestamp = timestamp;
         this.unit = unit;
     }
@@ -62,15 +74,17 @@ public Rate(StreamInput in) throws IOException {
         this(
             Source.readFrom((PlanStreamInput) in),
             in.readNamedWriteable(Expression.class),
-            in.readNamedWriteable(Expression.class),
-            in.readOptionalNamedWriteable(Expression.class)
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class)
+                : nullSafeList(in.readNamedWriteable(Expression.class), in.readOptionalNamedWriteable(Expression.class))
         );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        source().writeTo(out);
-        out.writeNamedWriteable(field());
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
         out.writeNamedWriteable(timestamp);
         out.writeOptionalNamedWriteable(unit);
     }
@@ -92,20 +106,25 @@ protected NodeInfo<Rate> info() {
     @Override
     public Rate replaceChildren(List<Expression> newChildren) {
         if (unit != null) {
-            if (newChildren.size() == 3) {
-                return new Rate(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+            if (newChildren.size() == 4) {
+                return new Rate(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3));
             }
-            assert false : "expected 3 children for field, @timestamp, and unit; got " + newChildren;
-            throw new IllegalArgumentException("expected 3 children for field, @timestamp, and unit; got " + newChildren);
+            assert false : "expected 4 children for field, filter, @timestamp, and unit; got " + newChildren;
+            throw new IllegalArgumentException("expected 4 children for field, filter, @timestamp, and unit; got " + newChildren);
         } else {
-            if (newChildren.size() == 2) {
-                return new Rate(source(), newChildren.get(0), newChildren.get(1), null);
+            if (newChildren.size() == 3) {
+                return new Rate(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), null);
             }
-            assert false : "expected 2 children for field and @timestamp; got " + newChildren;
-            throw new IllegalArgumentException("expected 2 children for field and @timestamp; got " + newChildren);
+            assert false : "expected 3 children for field, filter and @timestamp; got " + newChildren;
+            throw new IllegalArgumentException("expected 3 children for field, filter and @timestamp; got " + newChildren);
         }
     }
 
+    @Override
+    public Rate withFilter(Expression filter) {
+        return new Rate(source(), field(), filter, timestamp, unit);
+    }
+
     @Override
     public DataType dataType() {
         return DataType.DOUBLE;
@@ -115,7 +134,7 @@ public DataType dataType() {
     protected TypeResolution resolveType() {
         TypeResolution resolution = isType(
             field(),
-            dt -> dt == DataType.COUNTER_LONG || dt == DataType.COUNTER_INTEGER || dt == DataType.COUNTER_DOUBLE,
+            dt -> DataType.isCounter(dt),
             sourceText(),
             FIRST,
             "counter_long",
@@ -149,11 +168,6 @@ long unitInMillis() {
         throw new IllegalArgumentException("function [" + sourceText() + "] has invalid unit [" + unit.sourceText() + "]");
     }
 
-    @Override
-    public List<Expression> inputExpressions() {
-        return List.of(field(), timestamp);
-    }
-
     @Override
     public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         if (inputChannels.size() != 2 && inputChannels.size() != 3) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java
index 5cb7edf2581d5..87eec540932b1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java
@@ -14,6 +14,8 @@
 import java.io.IOException;
 import java.util.Objects;
 
+import static java.util.Collections.emptyList;
+
 /**
  * All spatial aggregate functions extend this class to enable the planning of reading from doc values for higher performance.
  * The AggregateMapper class will generate multiple aggregation functions for each combination, allowing the planner to
@@ -22,8 +24,8 @@
 public abstract class SpatialAggregateFunction extends AggregateFunction {
     protected final boolean useDocValues;
 
-    protected SpatialAggregateFunction(Source source, Expression field, boolean useDocValues) {
-        super(source, field);
+    protected SpatialAggregateFunction(Source source, Expression field, Expression filter, boolean useDocValues) {
+        super(source, field, filter, emptyList());
         this.useDocValues = useDocValues;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
index b9cd99f8eb7f0..aad95c07e3492 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.compute.aggregation.spatial.SpatialCentroidGeoPointSourceValuesAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -46,11 +47,11 @@ public class SpatialCentroid extends SpatialAggregateFunction implements ToAggre
         examples = @Example(file = "spatial", tag = "st_centroid_agg-airports")
     )
     public SpatialCentroid(Source source, @Param(name = "field", type = { "geo_point", "cartesian_point" }) Expression field) {
-        super(source, field, false);
+        this(source, field, Literal.TRUE, false);
     }
 
-    private SpatialCentroid(Source source, Expression field, boolean useDocValues) {
-        super(source, field, useDocValues);
+    private SpatialCentroid(Source source, Expression field, Expression filter, boolean useDocValues) {
+        super(source, field, filter, useDocValues);
     }
 
     private SpatialCentroid(StreamInput in) throws IOException {
@@ -62,9 +63,14 @@ public String getWriteableName() {
         return ENTRY.name;
     }
 
+    @Override
+    public SpatialCentroid withFilter(Expression filter) {
+        return new SpatialCentroid(source(), field(), filter, useDocValues);
+    }
+
     @Override
     public SpatialCentroid withDocValues() {
-        return new SpatialCentroid(source(), field(), true);
+        return new SpatialCentroid(source(), field(), filter(), true);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
index 4f85a15732a6f..37c2abaae1e4e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
@@ -53,7 +54,11 @@ public class Sum extends NumericAggregate implements SurrogateExpression {
             ) }
     )
     public Sum(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) {
-        super(source, field);
+        this(source, field, Literal.TRUE);
+    }
+
+    public Sum(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     private Sum(StreamInput in) throws IOException {
@@ -67,12 +72,17 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<Sum> info() {
-        return NodeInfo.create(this, Sum::new, field());
+        return NodeInfo.create(this, Sum::new, field(), filter());
     }
 
     @Override
     public Sum replaceChildren(List<Expression> newChildren) {
-        return new Sum(source(), newChildren.get(0));
+        return new Sum(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public Sum withFilter(Expression filter) {
+        return new Sum(source(), field(), filter);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java
index c1da400185944..cffac616b3c8c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -21,6 +22,7 @@
 import org.elasticsearch.compute.aggregation.ToPartialGroupingAggregatorFunction;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -66,18 +68,29 @@ public class ToPartial extends AggregateFunction implements ToAggregator {
     private final Expression function;
 
     public ToPartial(Source source, Expression field, Expression function) {
-        super(source, field, List.of(function));
+        this(source, field, Literal.TRUE, function);
+    }
+
+    public ToPartial(Source source, Expression field, Expression filter, Expression function) {
+        super(source, field, filter, List.of(function));
         this.function = function;
     }
 
     private ToPartial(StreamInput in) throws IOException {
-        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(Expression.class),
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class).get(0)
+                : in.readNamedWriteable(Expression.class)
+        );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        source().writeTo(out);
-        out.writeNamedWriteable(field());
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
         out.writeNamedWriteable(function);
     }
 
@@ -102,12 +115,17 @@ protected TypeResolution resolveType() {
 
     @Override
     public Expression replaceChildren(List<Expression> newChildren) {
-        return new ToPartial(source(), newChildren.get(0), newChildren.get(1));
+        return new ToPartial(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+
+    @Override
+    public ToPartial withFilter(Expression filter) {
+        return new ToPartial(source(), field(), filter(), function);
     }
 
     @Override
-    protected NodeInfo<? extends Expression> info() {
-        return NodeInfo.create(this, ToPartial::new, field(), function);
+    protected NodeInfo<ToPartial> info() {
+        return NodeInfo.create(this, ToPartial::new, field(), filter(), function);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
index cb1b0f0cad895..4f81e0a897f9c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -20,6 +21,7 @@
 import org.elasticsearch.compute.aggregation.TopLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -31,9 +33,9 @@
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.List;
 
+import static java.util.Arrays.asList;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
@@ -68,26 +70,37 @@ public Top(
             description = "The order to calculate the top values. Either `asc` or `desc`."
         ) Expression order
     ) {
-        super(source, field, Arrays.asList(limit, order));
+        this(source, field, Literal.TRUE, limit, order);
+    }
+
+    public Top(Source source, Expression field, Expression filter, Expression limit, Expression order) {
+        super(source, field, filter, asList(limit, order));
     }
 
     private Top(StreamInput in) throws IOException {
-        this(
+        super(
             Source.readFrom((PlanStreamInput) in),
             in.readNamedWriteable(Expression.class),
-            in.readNamedWriteable(Expression.class),
-            in.readNamedWriteable(Expression.class)
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class)
+                : asList(in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class))
         );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        source().writeTo(out);
-        List<Expression> fields = children();
-        assert fields.size() == 3;
-        out.writeNamedWriteable(fields.get(0));
-        out.writeNamedWriteable(fields.get(1));
-        out.writeNamedWriteable(fields.get(2));
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
+        List<? extends Expression> params = parameters();
+        assert params.size() == 2;
+        out.writeNamedWriteable(params.get(0));
+        out.writeNamedWriteable(params.get(1));
+    }
+
+    @Override
+    public Top withFilter(Expression filter) {
+        return new Top(source(), field(), filter, limitField(), orderField());
     }
 
     @Override
@@ -167,12 +180,12 @@ public DataType dataType() {
 
     @Override
     protected NodeInfo<Top> info() {
-        return NodeInfo.create(this, Top::new, children().get(0), children().get(1), children().get(2));
+        return NodeInfo.create(this, Top::new, field(), filter(), limitField(), orderField());
     }
 
     @Override
     public Top replaceChildren(List<Expression> newChildren) {
-        return new Top(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+        return new Top(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
index 136e1233601f9..8d576839c3c5c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.compute.aggregation.ValuesLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -28,13 +29,28 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 public class Values extends AggregateFunction implements ToAggregator {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Values", Values::new);
 
+    private static final Map<DataType, Function<List<Integer>, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries(
+        Map.entry(DataType.INTEGER, ValuesIntAggregatorFunctionSupplier::new),
+        Map.entry(DataType.LONG, ValuesLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATETIME, ValuesLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DATE_NANOS, ValuesLongAggregatorFunctionSupplier::new),
+        Map.entry(DataType.DOUBLE, ValuesDoubleAggregatorFunctionSupplier::new),
+        Map.entry(DataType.KEYWORD, ValuesBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.TEXT, ValuesBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.IP, ValuesBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.VERSION, ValuesBytesRefAggregatorFunctionSupplier::new),
+        Map.entry(DataType.BOOLEAN, ValuesBooleanAggregatorFunctionSupplier::new)
+    );
+
     @FunctionInfo(
         returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
         preview = true,
@@ -56,7 +72,11 @@ public Values(
         Source source,
         @Param(name = "field", type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }) Expression v
     ) {
-        super(source, v);
+        this(source, v, Literal.TRUE);
+    }
+
+    public Values(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
     }
 
     private Values(StreamInput in) throws IOException {
@@ -70,12 +90,17 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<Values> info() {
-        return NodeInfo.create(this, Values::new, field());
+        return NodeInfo.create(this, Values::new, field(), filter());
     }
 
     @Override
     public Values replaceChildren(List<Expression> newChildren) {
-        return new Values(source(), newChildren.get(0));
+        return new Values(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public Values withFilter(Expression filter) {
+        return new Values(source(), field(), filter);
     }
 
     @Override
@@ -87,7 +112,7 @@ public DataType dataType() {
     protected TypeResolution resolveType() {
         return TypeResolutions.isType(
             field(),
-            dt -> DataType.isSpatial(dt) == false && dt != UNSIGNED_LONG,
+            SUPPLIERS::containsKey,
             sourceText(),
             DEFAULT,
             "any type except unsigned_long and spatial types"
@@ -97,22 +122,10 @@ protected TypeResolution resolveType() {
     @Override
     public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
-        if (type == DataType.INTEGER) {
-            return new ValuesIntAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.LONG || type == DataType.DATETIME) {
-            return new ValuesLongAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.DOUBLE) {
-            return new ValuesDoubleAggregatorFunctionSupplier(inputChannels);
-        }
-        if (DataType.isString(type) || type == DataType.IP || type == DataType.VERSION) {
-            return new ValuesBytesRefAggregatorFunctionSupplier(inputChannels);
-        }
-        if (type == DataType.BOOLEAN) {
-            return new ValuesBooleanAggregatorFunctionSupplier(inputChannels);
+        if (SUPPLIERS.containsKey(type) == false) {
+            // If the type checking did its job, this should never happen
+            throw EsqlIllegalArgumentException.illegalDataType(type);
         }
-        // TODO cartesian_point, geo_point
-        throw EsqlIllegalArgumentException.illegalDataType(type);
+        return SUPPLIERS.get(type).apply(inputChannels);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
index 23a20d9897e72..dbcc50cea3b9b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
@@ -7,11 +7,13 @@
 
 package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.capabilities.Validatable;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -54,21 +56,30 @@ public WeightedAvg(
         @Param(name = "number", type = { "double", "integer", "long" }, description = "A numeric value.") Expression field,
         @Param(name = "weight", type = { "double", "integer", "long" }, description = "A numeric weight.") Expression weight
     ) {
-        super(source, field, List.of(weight));
+        this(source, field, Literal.TRUE, weight);
+    }
+
+    public WeightedAvg(Source source, Expression field, Expression filter, Expression weight) {
+        super(source, field, filter, List.of(weight));
         this.weight = weight;
     }
 
     private WeightedAvg(StreamInput in) throws IOException {
-        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(Expression.class),
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteable(Expression.class)
+                : Literal.TRUE,
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)
+                ? in.readNamedWriteableCollectionAsList(Expression.class).get(0)
+                : in.readNamedWriteable(Expression.class)
+        );
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        source().writeTo(out);
-        List<Expression> fields = children();
-        assert fields.size() == 2;
-        out.writeNamedWriteable(fields.get(0));
-        out.writeNamedWriteable(fields.get(1));
+    protected void deprecatedWriteParams(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(weight);
     }
 
     @Override
@@ -121,12 +132,17 @@ public DataType dataType() {
 
     @Override
     protected NodeInfo<WeightedAvg> info() {
-        return NodeInfo.create(this, WeightedAvg::new, field(), weight);
+        return NodeInfo.create(this, WeightedAvg::new, field(), filter(), weight);
     }
 
     @Override
     public WeightedAvg replaceChildren(List<Expression> newChildren) {
-        return new WeightedAvg(source(), newChildren.get(0), newChildren.get(1));
+        return new WeightedAvg(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+
+    @Override
+    public WeightedAvg withFilter(Expression filter) {
+        return new WeightedAvg(source(), field(), filter, weight());
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
index 54730eec4f317..2f97de4c64469 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
@@ -7,22 +7,18 @@
 
 package org.elasticsearch.xpack.esql.expression.function.fulltext;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
-import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
 
-import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 
-import static java.util.Collections.singletonList;
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
@@ -34,47 +30,79 @@
  */
 public abstract class FullTextFunction extends Function {
     public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
-        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-        if (EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()) {
-            entries.add(QueryStringFunction.ENTRY);
-        }
-        return entries;
+        return List.of(QueryString.ENTRY, Match.ENTRY);
     }
 
     private final Expression query;
 
-    protected FullTextFunction(Source source, Expression query) {
-        super(source, singletonList(query));
+    protected FullTextFunction(Source source, Expression query, List<Expression> children) {
+        super(source, children);
         this.query = query;
     }
 
-    protected FullTextFunction(StreamInput in) throws IOException {
-        this(Source.readFrom((StreamInput & PlanStreamInput) in), in.readNamedWriteable(Expression.class));
-    }
-
     @Override
     public DataType dataType() {
         return DataType.BOOLEAN;
     }
 
     @Override
-    protected TypeResolution resolveType() {
+    protected final TypeResolution resolveType() {
         if (childrenResolved() == false) {
             return new TypeResolution("Unresolved children");
         }
 
-        return isString(query(), sourceText(), DEFAULT).and(isNotNullAndFoldable(query(), functionName(), DEFAULT));
+        return resolveNonQueryParamTypes().and(resolveQueryParamType());
+    }
+
+    /**
+     * Resolves the type for the query parameter, as part of the type resolution for the function
+     *
+     * @return type resolution for query parameter
+     */
+    private TypeResolution resolveQueryParamType() {
+        return isString(query(), sourceText(), queryParamOrdinal()).and(isNotNullAndFoldable(query(), sourceText(), queryParamOrdinal()));
+    }
+
+    /**
+     * Subclasses can override this method for custom type resolution for additional function parameters
+     *
+     * @return type resolution for non-query parameter types
+     */
+    protected TypeResolution resolveNonQueryParamTypes() {
+        return TypeResolution.TYPE_RESOLVED;
     }
 
     public Expression query() {
         return query;
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        source().writeTo(out);
-        out.writeNamedWriteable(query);
+    /**
+     * Returns the resulting query as a String
+     *
+     * @return query expression as a string
+     */
+    public final String queryAsText() {
+        Object queryAsObject = query().fold();
+        if (queryAsObject instanceof BytesRef bytesRef) {
+            return bytesRef.utf8ToString();
+        }
+
+        throw new IllegalArgumentException(
+            format(null, "{} argument in {} function needs to be resolved to a string", queryParamOrdinal(), functionName())
+        );
     }
 
-    public abstract Query asQuery();
+    /**
+     * Returns the param ordinal for the query parameter so it can be used in error messages
+     *
+     * @return Query ordinal for the
+     */
+    protected TypeResolutions.ParamOrdinal queryParamOrdinal() {
+        return DEFAULT;
+    }
+
+    @Override
+    public Nullability nullable() {
+        return Nullability.FALSE;
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java
new file mode 100644
index 0000000000000..b4e0f3c743216
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.fulltext;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.Validatable;
+import org.elasticsearch.xpack.esql.common.Failure;
+import org.elasticsearch.xpack.esql.common.Failures;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
+
+/**
+ * Full text function that performs a {@link QueryStringQuery} .
+ */
+public class Match extends FullTextFunction implements Validatable {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Match", Match::new);
+
+    private final Expression field;
+
+    @FunctionInfo(
+        returnType = "boolean",
+        preview = true,
+        description = "Performs a match query on the specified field. Returns true if the provided query matches the row.",
+        examples = { @Example(file = "match-function", tag = "match-with-field") }
+    )
+    public Match(
+        Source source,
+        @Param(name = "field", type = { "keyword", "text" }, description = "Field that the query will target.") Expression field,
+        @Param(
+            name = "query",
+            type = { "keyword", "text" },
+            description = "Text you wish to find in the provided field."
+        ) Expression matchQuery
+    ) {
+        super(source, matchQuery, List.of(field, matchQuery));
+        this.field = field;
+    }
+
+    private Match(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteable(field);
+        out.writeNamedWriteable(query());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    protected TypeResolution resolveNonQueryParamTypes() {
+        return isNotNull(field, sourceText(), FIRST).and(isString(field, sourceText(), FIRST)).and(super.resolveNonQueryParamTypes());
+    }
+
+    @Override
+    public void validate(Failures failures) {
+        if (field instanceof FieldAttribute == false) {
+            failures.add(
+                Failure.fail(
+                    field,
+                    "[{}] cannot operate on [{}], which is not a field from an index mapping",
+                    functionName(),
+                    field.sourceText()
+                )
+            );
+        }
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        // Query is the first child, field is the second child
+        return new Match(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Match::new, field, query());
+    }
+
+    protected TypeResolutions.ParamOrdinal queryParamOrdinal() {
+        return SECOND;
+    }
+
+    public Expression field() {
+        return field;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java
similarity index 66%
rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java
index fa331acd08655..0d7d15a13dd80 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java
@@ -7,32 +7,27 @@
 
 package org.elasticsearch.xpack.esql.expression.function.fulltext;
 
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Map;
 
 /**
  * Full text function that performs a {@link QueryStringQuery} .
  */
-public class QueryStringFunction extends FullTextFunction {
+public class QueryString extends FullTextFunction {
 
-    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
-        Expression.class,
-        "QStr",
-        QueryStringFunction::new
-    );
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "QStr", QueryString::new);
 
     @FunctionInfo(
         returnType = "boolean",
@@ -40,7 +35,7 @@ public class QueryStringFunction extends FullTextFunction {
         description = "Performs a query string query. Returns true if the provided query string matches the row.",
         examples = { @Example(file = "qstr-function", tag = "qstr-with-field") }
     )
-    public QueryStringFunction(
+    public QueryString(
         Source source,
         @Param(
             name = "query",
@@ -48,40 +43,37 @@ public QueryStringFunction(
             description = "Query string in Lucene query string format."
         ) Expression queryString
     ) {
-        super(source, queryString);
+        super(source, queryString, List.of(queryString));
     }
 
-    private QueryStringFunction(StreamInput in) throws IOException {
-        super(in);
+    private QueryString(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class));
     }
 
     @Override
-    public String functionName() {
-        return "QSTR";
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteable(query());
     }
 
     @Override
-    public Query asQuery() {
-        Object queryAsObject = query().fold();
-        if (queryAsObject instanceof BytesRef queryAsBytesRef) {
-            return new QueryStringQuery(source(), queryAsBytesRef.utf8ToString(), Map.of(), null);
-        } else {
-            throw new IllegalArgumentException("Query in QSTR needs to be resolved to a string");
-        }
+    public String getWriteableName() {
+        return ENTRY.name;
     }
 
     @Override
-    public Expression replaceChildren(List<Expression> newChildren) {
-        return new QueryStringFunction(source(), newChildren.get(0));
+    public String functionName() {
+        return "QSTR";
     }
 
     @Override
-    protected NodeInfo<? extends Expression> info() {
-        return NodeInfo.create(this, QueryStringFunction::new, query());
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new QueryString(source(), newChildren.get(0));
     }
 
     @Override
-    public String getWriteableName() {
-        return ENTRY.name;
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, QueryString::new, query());
     }
+
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
index afe9bf6e45eda..e4e1fbb6e5aac 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Atan2;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.E;
+import org.elasticsearch.xpack.esql.expression.function.scalar.math.Hypot;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pi;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow;
@@ -82,6 +83,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(E.ENTRY);
         entries.add(EndsWith.ENTRY);
         entries.add(Greatest.ENTRY);
+        entries.add(Hypot.ENTRY);
         entries.add(In.ENTRY);
         entries.add(InsensitiveEquals.ENTRY);
         entries.add(DateExtract.ENTRY);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java
index 9d815d15accdc..aad2d37d414b8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java
@@ -43,7 +43,7 @@ public class Greatest extends EsqlScalarFunction implements OptionalArgument {
     private DataType dataType;
 
     @FunctionInfo(
-        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+        returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
         description = "Returns the maximum value from multiple columns. This is similar to <<esql-mv_max>>\n"
             + "except it is intended to run on multiple columns at once.",
         note = "When run on `keyword` or `text` fields, this returns the last string in alphabetical order. "
@@ -54,12 +54,12 @@ public Greatest(
         Source source,
         @Param(
             name = "first",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
             description = "First of the columns to evaluate."
         ) Expression first,
         @Param(
             name = "rest",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
             description = "The rest of the columns to evaluate.",
             optional = true
         ) List<Expression> rest
@@ -152,14 +152,10 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (dataType == DataType.INTEGER) {
             return new GreatestIntEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataType.LONG || dataType == DataType.DATETIME) {
+        if (dataType == DataType.LONG || dataType == DataType.DATETIME || dataType == DataType.DATE_NANOS) {
             return new GreatestLongEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataType.KEYWORD
-            || dataType == DataType.TEXT
-            || dataType == DataType.IP
-            || dataType == DataType.VERSION
-            || dataType == DataType.UNSUPPORTED) {
+        if (DataType.isString(dataType) || dataType == DataType.IP || dataType == DataType.VERSION || dataType == DataType.UNSUPPORTED) {
 
             return new GreatestBytesRefEvaluator.Factory(source(), factories);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java
index 435a14d0fef33..70ba9319385f3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java
@@ -43,7 +43,7 @@ public class Least extends EsqlScalarFunction implements OptionalArgument {
     private DataType dataType;
 
     @FunctionInfo(
-        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+        returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
         description = "Returns the minimum value from multiple columns. "
             + "This is similar to <<esql-mv_min>> except it is intended to run on multiple columns at once.",
         examples = @Example(file = "math", tag = "least")
@@ -52,12 +52,12 @@ public Least(
         Source source,
         @Param(
             name = "first",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
             description = "First of the columns to evaluate."
         ) Expression first,
         @Param(
             name = "rest",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
             description = "The rest of the columns to evaluate.",
             optional = true
         ) List<Expression> rest
@@ -151,14 +151,10 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (dataType == DataType.INTEGER) {
             return new LeastIntEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataType.LONG || dataType == DataType.DATETIME) {
+        if (dataType == DataType.LONG || dataType == DataType.DATETIME || dataType == DataType.DATE_NANOS) {
             return new LeastLongEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataType.KEYWORD
-            || dataType == DataType.TEXT
-            || dataType == DataType.IP
-            || dataType == DataType.VERSION
-            || dataType == DataType.UNSUPPORTED) {
+        if (DataType.isString(dataType) || dataType == DataType.IP || dataType == DataType.VERSION || dataType == DataType.UNSUPPORTED) {
 
             return new LeastBytesRefEvaluator.Factory(source(), factories);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
index 5401fcf188d4a..06815d738e82c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
@@ -48,7 +48,6 @@ public abstract class AbstractConvertFunction extends UnaryScalarFunction {
 
     // the numeric types convert functions need to handle; the other numeric types are converted upstream to one of these
     private static final List<DataType> NUMERIC_TYPES = List.of(DataType.INTEGER, DataType.LONG, DataType.UNSIGNED_LONG, DataType.DOUBLE);
-    public static final List<DataType> STRING_TYPES = DataType.types().stream().filter(DataType::isString).toList();
 
     protected AbstractConvertFunction(Source source, Expression field) {
         super(source, field);
@@ -90,9 +89,9 @@ private static String supportedTypesNames(Set<DataType> types) {
             NUMERIC_TYPES.forEach(supportTypes::remove);
         }
 
-        if (types.containsAll(STRING_TYPES)) {
+        if (types.containsAll(DataType.stringTypes())) {
             supportedTypesNames.add("string");
-            STRING_TYPES.forEach(supportTypes::remove);
+            DataType.stringTypes().forEach(supportTypes::remove);
         }
 
         supportTypes.forEach(t -> supportedTypesNames.add(t.nameUpper().toLowerCase(Locale.ROOT)));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Hypot.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Hypot.java
new file mode 100644
index 0000000000000..1a644c929f3c3
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Hypot.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric;
+
+/**
+ * Returns the hypotenuse of the numbers given as parameters.
+ */
+public class Hypot extends EsqlScalarFunction {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Hypot", Hypot::new);
+
+    private final Expression n1;
+    private final Expression n2;
+
+    @FunctionInfo(returnType = "double", description = """
+        Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double.
+        Hypotenuses of infinities are null.""", examples = @Example(file = "math", tag = "hypot"))
+    public Hypot(
+        Source source,
+        @Param(
+            name = "number1",
+            type = { "double", "integer", "long", "unsigned_long" },
+            description = "Numeric expression. If `null`, the function returns `null`."
+        ) Expression n1,
+        @Param(
+            name = "number2",
+            type = { "double", "integer", "long", "unsigned_long" },
+            description = "Numeric expression. If `null`, the function returns `null`."
+        ) Expression n2
+    ) {
+        super(source, List.of(n1, n2));
+        this.n1 = n1;
+        this.n2 = n2;
+    }
+
+    private Hypot(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteable(n1);
+        out.writeNamedWriteable(n2);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Hypot(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Hypot::new, n1, n2);
+    }
+
+    @Evaluator
+    static double process(double n1, double n2) {
+        return Math.hypot(n1, n2);
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.DOUBLE;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = isNumeric(n1, sourceText(), TypeResolutions.ParamOrdinal.FIRST);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        return isNumeric(n2, sourceText(), TypeResolutions.ParamOrdinal.SECOND);
+    }
+
+    @Override
+    public boolean foldable() {
+        return Expressions.foldable(children());
+    }
+
+    @Override
+    public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
+        var n1Eval = Cast.cast(source(), n1.dataType(), DataType.DOUBLE, toEvaluator.apply(n1));
+        var n2Eval = Cast.cast(source(), n2.dataType(), DataType.DOUBLE, toEvaluator.apply(n2));
+        return new HypotEvaluator.Factory(source(), n1Eval, n2Eval);
+    }
+
+    public Expression n1() {
+        return n1;
+    }
+
+    public Expression n2() {
+        return n2;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
index b17ddddb422ce..34b89b4f78997 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
@@ -38,6 +38,7 @@ public class MvDedupe extends AbstractMultivalueFunction {
             "cartesian_point",
             "cartesian_shape",
             "date",
+            "date_nanos",
             "double",
             "geo_point",
             "geo_shape",
@@ -60,6 +61,7 @@ public MvDedupe(
                 "cartesian_point",
                 "cartesian_shape",
                 "date",
+                "date_nanos",
                 "double",
                 "geo_point",
                 "geo_shape",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
index a829b6f1417b9..ef562c339dfd9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
@@ -59,6 +59,7 @@ public class MvSlice extends EsqlScalarFunction implements OptionalArgument, Eva
             "cartesian_point",
             "cartesian_shape",
             "date",
+            "date_nanos",
             "double",
             "geo_point",
             "geo_shape",
@@ -87,6 +88,7 @@ public MvSlice(
                 "cartesian_point",
                 "cartesian_shape",
                 "date",
+                "date_nanos",
                 "double",
                 "geo_point",
                 "geo_shape",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
index d9e41233952de..5ca5618bf2a54 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
@@ -69,7 +69,7 @@ public class MvSort extends EsqlScalarFunction implements OptionalArgument, Vali
     private static final String INVALID_ORDER_ERROR = "Invalid order value in [{}], expected one of [{}, {}] but got [{}]";
 
     @FunctionInfo(
-        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+        returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
         description = "Sorts a multivalued field in lexicographical order.",
         examples = @Example(file = "ints", tag = "mv_sort")
     )
@@ -77,7 +77,7 @@ public MvSort(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" },
             description = "Multivalue expression. If `null`, the function returns `null`."
         ) Expression field,
         @Param(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java
index 575bb085c41f7..6b9c8d0da025b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java
@@ -53,6 +53,7 @@ public class Coalesce extends EsqlScalarFunction implements OptionalArgument {
             "boolean",
             "cartesian_point",
             "cartesian_shape",
+            "date_nanos",
             "date",
             "geo_point",
             "geo_shape",
@@ -73,6 +74,7 @@ public Coalesce(
                 "boolean",
                 "cartesian_point",
                 "cartesian_shape",
+                "date_nanos",
                 "date",
                 "geo_point",
                 "geo_shape",
@@ -90,6 +92,7 @@ public Coalesce(
                 "boolean",
                 "cartesian_point",
                 "cartesian_shape",
+                "date_nanos",
                 "date",
                 "geo_point",
                 "geo_shape",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
index 5e8d39217fcca..8839244e6c601 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
@@ -28,6 +28,8 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
@@ -203,7 +205,7 @@ public void setCrsType(DataType dataType) {
     }
 
     private static final String[] GEO_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() };
-    private static final String[] CARTESIAN_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() };
+    private static final String[] CARTESIAN_TYPE_NAMES = new String[] { CARTESIAN_POINT.typeName(), CARTESIAN_SHAPE.typeName() };
 
     protected static boolean spatialCRSCompatible(DataType spatialDataType, DataType otherDataType) {
         return DataType.isSpatialGeo(spatialDataType) && DataType.isSpatialGeo(otherDataType)
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
index ae9d3383bad39..3cf042a2db828 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
@@ -14,6 +14,9 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Point;
@@ -77,8 +80,9 @@ protected CartesianDistanceCalculator() {
 
         @Override
         protected double distance(Point left, Point right) {
-            final double diffX = left.getX() - right.getX();
-            final double diffY = left.getY() - right.getY();
+            // Cast coordinates to float to mimic Lucene behaviour, so we get identical results
+            final double diffX = (double) ((float) left.getX()) - (double) ((float) right.getX());
+            final double diffY = (double) ((float) left.getY()) - (double) ((float) right.getY());
             return Math.sqrt(diffX * diffX + diffY * diffY);
         }
     }
@@ -100,11 +104,6 @@ protected Double compare(BytesRef left, BytesRef right) throws IOException {
 
         protected abstract double distance(Point left, Point right);
 
-        protected double distance(long encoded, Geometry right) {
-            Point point = spatialCoordinateType.longAsPoint(encoded);
-            return distance(point, (Point) right);
-        }
-
         protected double distance(Geometry left, Geometry right) {
             return distance((Point) left, (Point) right);
         }
@@ -113,8 +112,112 @@ public double distance(BytesRef left, BytesRef right) {
             return distance(this.fromBytesRef(left), this.fromBytesRef(right));
         }
 
-        public double distance(BytesRef left, Point right) {
-            return distance(this.fromBytesRef(left), right);
+        public void distanceSourceAndConstant(DoubleBlock.Builder results, int position, BytesRefBlock left, Point right) {
+            int valueCount = left.getValueCount(position);
+            if (valueCount < 1) {
+                results.appendNull();
+            } else {
+                final BytesRef scratch = new BytesRef();
+                final int firstValueIndex = left.getFirstValueIndex(position);
+                double distance = Double.MAX_VALUE;
+                if (valueCount == 1) {
+                    distance = distance(fromBytesRef(left.getBytesRef(firstValueIndex, scratch)), right);
+                } else {
+                    for (int i = 0; i < valueCount; i++) {
+                        double value = distance(fromBytesRef(left.getBytesRef(firstValueIndex + i, scratch)), right);
+                        if (value < distance) {
+                            distance = value;
+                        }
+                    }
+                }
+                results.appendDouble(distance);
+            }
+        }
+
+        public void distanceSourceAndSource(DoubleBlock.Builder results, int position, BytesRefBlock left, BytesRefBlock right) {
+            int leftCount = left.getValueCount(position);
+            int rightCount = right.getValueCount(position);
+            if (leftCount < 1 || rightCount < 1) {
+                results.appendNull();
+            } else {
+                final BytesRef scratchLeft = new BytesRef();
+                final BytesRef scratchRight = new BytesRef();
+                final int leftFirstValueIndex = left.getFirstValueIndex(position);
+                final int rightFirstValueIndex = right.getFirstValueIndex(position);
+                double distance = Double.MAX_VALUE;
+                if (leftCount == 1 && rightCount == 1) {
+                    distance = distance(
+                        fromBytesRef(left.getBytesRef(leftFirstValueIndex, scratchLeft)),
+                        fromBytesRef(right.getBytesRef(rightFirstValueIndex, scratchRight))
+                    );
+                } else {
+                    for (int i = 0; i < leftCount; i++) {
+                        for (int j = 0; j < rightCount; j++) {
+                            double value = distance(
+                                fromBytesRef(left.getBytesRef(leftFirstValueIndex + i, scratchLeft)),
+                                fromBytesRef(right.getBytesRef(rightFirstValueIndex + j, scratchRight))
+                            );
+                            if (value < distance) {
+                                distance = value;
+                            }
+                        }
+                    }
+                }
+                results.appendDouble(distance);
+            }
+        }
+
+        public void distancePointDocValuesAndConstant(DoubleBlock.Builder results, int position, LongBlock left, Point right) {
+            int valueCount = left.getValueCount(position);
+            if (valueCount < 1) {
+                results.appendNull();
+            } else {
+                final int firstValueIndex = left.getFirstValueIndex(position);
+                double distance = Double.MAX_VALUE;
+                if (valueCount == 1) {
+                    distance = distance(spatialCoordinateType.longAsPoint(left.getLong(firstValueIndex)), right);
+                } else {
+                    for (int i = 0; i < valueCount; i++) {
+                        double value = distance(spatialCoordinateType.longAsPoint(left.getLong(firstValueIndex + i)), right);
+                        if (value < distance) {
+                            distance = value;
+                        }
+                    }
+                }
+                results.appendDouble(distance);
+            }
+        }
+
+        public void distancePointDocValuesAndSource(DoubleBlock.Builder results, int position, LongBlock left, BytesRefBlock right) {
+            int leftCount = left.getValueCount(position);
+            int rightCount = right.getValueCount(position);
+            if (leftCount < 1 || rightCount < 1) {
+                results.appendNull();
+            } else {
+                final BytesRef scratchRight = new BytesRef();
+                final int leftFirstValueIndex = left.getFirstValueIndex(position);
+                final int rightFirstValueIndex = right.getFirstValueIndex(position);
+                double distance = Double.MAX_VALUE;
+                if (leftCount == 1 && rightCount == 1) {
+                    distance = distance(
+                        spatialCoordinateType.longAsPoint(left.getLong(leftFirstValueIndex)),
+                        fromBytesRef(right.getBytesRef(rightFirstValueIndex, scratchRight))
+                    );
+                }
+                for (int i = 0; i < leftCount; i++) {
+                    for (int j = 0; j < rightCount; j++) {
+                        double value = distance(
+                            spatialCoordinateType.longAsPoint(left.getLong(leftFirstValueIndex + i)),
+                            fromBytesRef(right.getBytesRef(rightFirstValueIndex + j, scratchRight))
+                        );
+                        if (value < distance) {
+                            distance = value;
+                        }
+                    }
+                }
+                results.appendDouble(distance);
+            }
+
         }
     }
 
@@ -249,45 +352,43 @@ private EvalOperator.ExpressionEvaluator.Factory toEvaluator(
         throw EsqlIllegalArgumentException.illegalDataType(crsType().name());
     }
 
-    @Evaluator(extraName = "GeoSourceAndConstant", warnExceptions = { IllegalArgumentException.class, IOException.class })
-    static double processGeoSourceAndConstant(BytesRef leftValue, @Fixed Point rightValue) throws IOException {
-        return GEO.distance(leftValue, rightValue);
+    @Evaluator(extraName = "GeoSourceAndConstant", warnExceptions = { IllegalArgumentException.class })
+    static void processGeoSourceAndConstant(DoubleBlock.Builder results, int p, BytesRefBlock left, @Fixed Point right) {
+        GEO.distanceSourceAndConstant(results, p, left, right);
     }
 
-    @Evaluator(extraName = "GeoSourceAndSource", warnExceptions = { IllegalArgumentException.class, IOException.class })
-    static double processGeoSourceAndSource(BytesRef leftValue, BytesRef rightValue) throws IOException {
-        return GEO.distance(leftValue, rightValue);
+    @Evaluator(extraName = "GeoSourceAndSource", warnExceptions = { IllegalArgumentException.class })
+    static void processGeoSourceAndSource(DoubleBlock.Builder results, int p, BytesRefBlock left, BytesRefBlock right) {
+        GEO.distanceSourceAndSource(results, p, left, right);
     }
 
     @Evaluator(extraName = "GeoPointDocValuesAndConstant", warnExceptions = { IllegalArgumentException.class })
-    static double processGeoPointDocValuesAndConstant(long leftValue, @Fixed Point rightValue) {
-        return GEO.distance(leftValue, rightValue);
+    static void processGeoPointDocValuesAndConstant(DoubleBlock.Builder results, int p, LongBlock left, @Fixed Point right) {
+        GEO.distancePointDocValuesAndConstant(results, p, left, right);
     }
 
     @Evaluator(extraName = "GeoPointDocValuesAndSource", warnExceptions = { IllegalArgumentException.class })
-    static double processGeoPointDocValuesAndSource(long leftValue, BytesRef rightValue) {
-        Geometry geometry = SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(rightValue);
-        return GEO.distance(leftValue, geometry);
+    static void processGeoPointDocValuesAndSource(DoubleBlock.Builder results, int p, LongBlock left, BytesRefBlock right) {
+        GEO.distancePointDocValuesAndSource(results, p, left, right);
     }
 
-    @Evaluator(extraName = "CartesianSourceAndConstant", warnExceptions = { IllegalArgumentException.class, IOException.class })
-    static double processCartesianSourceAndConstant(BytesRef leftValue, @Fixed Point rightValue) throws IOException {
-        return CARTESIAN.distance(leftValue, rightValue);
+    @Evaluator(extraName = "CartesianSourceAndConstant", warnExceptions = { IllegalArgumentException.class })
+    static void processCartesianSourceAndConstant(DoubleBlock.Builder results, int p, BytesRefBlock left, @Fixed Point right) {
+        CARTESIAN.distanceSourceAndConstant(results, p, left, right);
     }
 
-    @Evaluator(extraName = "CartesianSourceAndSource", warnExceptions = { IllegalArgumentException.class, IOException.class })
-    static double processCartesianSourceAndSource(BytesRef leftValue, BytesRef rightValue) throws IOException {
-        return CARTESIAN.distance(leftValue, rightValue);
+    @Evaluator(extraName = "CartesianSourceAndSource", warnExceptions = { IllegalArgumentException.class })
+    static void processCartesianSourceAndSource(DoubleBlock.Builder results, int p, BytesRefBlock left, BytesRefBlock right) {
+        CARTESIAN.distanceSourceAndSource(results, p, left, right);
     }
 
     @Evaluator(extraName = "CartesianPointDocValuesAndConstant", warnExceptions = { IllegalArgumentException.class })
-    static double processCartesianPointDocValuesAndConstant(long leftValue, @Fixed Point rightValue) {
-        return CARTESIAN.distance(leftValue, rightValue);
+    static void processCartesianPointDocValuesAndConstant(DoubleBlock.Builder results, int p, LongBlock left, @Fixed Point right) {
+        CARTESIAN.distancePointDocValuesAndConstant(results, p, left, right);
     }
 
     @Evaluator(extraName = "CartesianPointDocValuesAndSource")
-    static double processCartesianPointDocValuesAndSource(long leftValue, BytesRef rightValue) {
-        Geometry geometry = SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(rightValue);
-        return CARTESIAN.distance(leftValue, geometry);
+    static void processCartesianPointDocValuesAndSource(DoubleBlock.Builder results, int p, LongBlock left, BytesRefBlock right) {
+        CARTESIAN.distancePointDocValuesAndSource(results, p, left, right);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java
index 09166f0cff7a8..0af22a357aeca 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java
@@ -37,7 +37,7 @@ public static EvalOperator.ExpressionEvaluator.Factory toEvaluator(
          * we couldn't get a nice toDot - so we call UTF32ToUTF8 ourselves.
          */
         Automaton automaton = Operations.determinize(new UTF32ToUTF8().convert(utf32Automaton), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
-        ByteRunAutomaton run = new ByteRunAutomaton(automaton, true, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        ByteRunAutomaton run = new ByteRunAutomaton(automaton, true);
         return new AutomataMatchEvaluator.Factory(source, field, run, toDot(automaton));
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
index b46c46c89deba..cd42711177510 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
@@ -33,7 +33,23 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
         Use `RLIKE` to filter data based on string patterns using using
         <<regexp-syntax,regular expressions>>. `RLIKE` usually acts on a field placed on
         the left-hand side of the operator, but it can also act on a constant (literal)
-        expression. The right-hand side of the operator represents the pattern.""", examples = @Example(file = "docs", tag = "rlike"))
+        expression. The right-hand side of the operator represents the pattern.""", detailedDescription = """
+        Matching special characters (eg. `.`, `*`, `(`...) will require escaping.
+        The escape character is backslash `\\`. Since also backslash is a special character in string literals,
+        it will require further escaping.
+
+        [source.merge.styled,esql]
+        ----
+        include::{esql-specs}/string.csv-spec[tag=rlikeEscapingSingleQuotes]
+        ----
+
+        To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
+
+        [source.merge.styled,esql]
+        ----
+        include::{esql-specs}/string.csv-spec[tag=rlikeEscapingTripleQuotes]
+        ----
+        """, examples = @Example(file = "docs", tag = "rlike"))
     public RLike(
         Source source,
         @Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java
index bf4e47d8d0de4..e161566838cd9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java
@@ -10,7 +10,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -79,8 +78,6 @@ protected TypeResolution resolveType() {
 
     /**
      * Reverses a unicode string, keeping grapheme clusters together
-     * @param str
-     * @return
      */
     public static String reverseStringWithUnicodeCharacters(String str) {
         BreakIterator boundary = BreakIterator.getCharacterInstance(Locale.ROOT);
@@ -100,10 +97,12 @@ public static String reverseStringWithUnicodeCharacters(String str) {
         return reversed.toString();
     }
 
-    private static boolean isOneByteUTF8(BytesRef ref) {
+    private static boolean reverseBytesIsReverseUnicode(BytesRef ref) {
         int end = ref.offset + ref.length;
         for (int i = ref.offset; i < end; i++) {
-            if (ref.bytes[i] < 0) {
+            if (ref.bytes[i] < 0 // Anything encoded in multibyte utf-8
+                || ref.bytes[i] == 0x28 // Backspace
+            ) {
                 return false;
             }
         }
@@ -112,13 +111,13 @@ private static boolean isOneByteUTF8(BytesRef ref) {
 
     @Evaluator
     static BytesRef process(BytesRef val) {
-        if (isOneByteUTF8(val)) {
+        if (reverseBytesIsReverseUnicode(val)) {
             // this is the fast path. we know we can just reverse the bytes.
             BytesRef reversed = BytesRef.deepCopyOf(val);
             reverseArray(reversed.bytes, reversed.offset, reversed.length);
             return reversed;
         }
-        return BytesRefs.toBytesRef(reverseStringWithUnicodeCharacters(val.utf8ToString()));
+        return new BytesRef(reverseStringWithUnicodeCharacters(val.utf8ToString()));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
index 714c4ca04a862..c1b4f20f41795 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
@@ -43,7 +43,23 @@ also act on a constant (literal) expression. The right-hand side of the operator
         The following wildcard characters are supported:
 
         * `*` matches zero or more characters.
-        * `?` matches one character.""", examples = @Example(file = "docs", tag = "like"))
+        * `?` matches one character.""", detailedDescription = """
+        Matching the exact characters `*` and `.` will require escaping.
+        The escape character is backslash `\\`. Since also backslash is a special character in string literals,
+        it will require further escaping.
+
+        [source.merge.styled,esql]
+        ----
+        include::{esql-specs}/string.csv-spec[tag=likeEscapingSingleQuotes]
+        ----
+
+        To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
+
+        [source.merge.styled,esql]
+        ----
+        include::{esql-specs}/string.csv-spec[tag=likeEscapingTripleQuotes]
+        ----
+        """, examples = @Example(file = "docs", tag = "like"))
     public WildcardLike(
         Source source,
         @Param(name = "str", type = { "keyword", "text" }, description = "A literal expression.") Expression left,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
index d11f5c9b68532..f5704239993f9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
@@ -34,7 +34,7 @@ public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout
 
         var leftEval = toEvaluator(bc.left(), layout);
         var rightEval = toEvaluator(bc.right(), layout);
-        if (leftType == DataType.KEYWORD || leftType == DataType.TEXT) {
+        if (DataType.isString(leftType)) {
             if (bc.right().foldable() && DataType.isString(rightType)) {
                 BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold());
                 Automaton automaton = InsensitiveEquals.automaton(rightVal);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index c832f64363048..1e1cc3b86a9d5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -37,6 +37,8 @@
 import java.util.Map;
 import java.util.function.LongFunction;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
+
 /**
  * A customized stream input used to deserialize ESQL physical plan fragments. Complements stream
  * input with methods that read plan nodes, Attributes, Expressions, etc.
@@ -180,7 +182,7 @@ public NameId mapNameId(long l) {
     @SuppressWarnings("unchecked")
     public <A extends Attribute> A readAttributeWithCache(CheckedFunction<StreamInput, A, IOException> constructor) throws IOException {
         if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION)
-            || getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)) {
+            || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) {
             // it's safe to cast to int, since the max value for this is {@link PlanStreamOutput#MAX_SERIALIZED_ATTRIBUTES}
             int cacheId = Math.toIntExact(readZLong());
             if (cacheId < 0) {
@@ -220,11 +222,11 @@ private void cacheAttribute(int id, Attribute attr) {
     @SuppressWarnings("unchecked")
     public <A extends EsField> A readEsFieldWithCache() throws IOException {
         if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)
-            || getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)) {
+            || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) {
             // it's safe to cast to int, since the max value for this is {@link PlanStreamOutput#MAX_SERIALIZED_ATTRIBUTES}
             int cacheId = Math.toIntExact(readZLong());
             if (cacheId < 0) {
-                String className = readCachedString();
+                String className = readCachedStringWithVersionCheck(this);
                 Writeable.Reader<? extends EsField> reader = EsField.getReader(className);
                 cacheId = -1 - cacheId;
                 EsField result = reader.read(this);
@@ -234,7 +236,7 @@ public <A extends EsField> A readEsFieldWithCache() throws IOException {
                 return (A) esFieldFromCache(cacheId);
             }
         } else {
-            String className = readCachedString();
+            String className = readCachedStringWithVersionCheck(this);
             Writeable.Reader<? extends EsField> reader = EsField.getReader(className);
             return (A) reader.read(this);
         }
@@ -245,9 +247,6 @@ public <A extends EsField> A readEsFieldWithCache() throws IOException {
      */
     @Override
     public String readCachedString() throws IOException {
-        if (getTransportVersion().before(TransportVersions.ESQL_CACHED_STRING_SERIALIZATION)) {
-            return readString();
-        }
         int cacheId = Math.toIntExact(readZLong());
         if (cacheId < 0) {
             String string = readString();
@@ -259,6 +258,11 @@ public String readCachedString() throws IOException {
         }
     }
 
+    @Override
+    public String readOptionalCachedString() throws IOException {
+        return readBoolean() ? readCachedString() : null;
+    }
+
     private EsField esFieldFromCache(int id) throws IOException {
         EsField field = esFieldsCache[id];
         if (field == null) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
index 5e31a57ed669b..615c4266620c7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
@@ -30,6 +30,8 @@
 import java.util.IdentityHashMap;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
+
 /**
  * A customized stream output used to serialize ESQL physical plan fragments. Complements stream
  * output with methods that write plan nodes, Attributes, Expressions, etc.
@@ -153,7 +155,7 @@ public void writeCachedBlock(Block block) throws IOException {
     @Override
     public boolean writeAttributeCacheHeader(Attribute attribute) throws IOException {
         if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION)
-            || getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)) {
+            || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) {
             Integer cacheId = attributeIdFromCache(attribute);
             if (cacheId != null) {
                 writeZLong(cacheId);
@@ -185,7 +187,7 @@ private int cacheAttribute(Attribute attr) {
     @Override
     public boolean writeEsFieldCacheHeader(EsField field) throws IOException {
         if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)
-            || getTransportVersion().isPatchFrom(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15)) {
+            || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) {
             Integer cacheId = esFieldIdFromCache(field);
             if (cacheId != null) {
                 writeZLong(cacheId);
@@ -195,7 +197,7 @@ public boolean writeEsFieldCacheHeader(EsField field) throws IOException {
             cacheId = cacheEsField(field);
             writeZLong(-1 - cacheId);
         }
-        writeCachedString(field.getWriteableName());
+        writeCachedStringWithVersionCheck(this, field.getWriteableName());
         return true;
     }
 
@@ -207,10 +209,6 @@ public boolean writeEsFieldCacheHeader(EsField field) throws IOException {
      */
     @Override
     public void writeCachedString(String string) throws IOException {
-        if (getTransportVersion().before(TransportVersions.ESQL_CACHED_STRING_SERIALIZATION)) {
-            writeString(string);
-            return;
-        }
         Integer cacheId = stringCache.get(string);
         if (cacheId != null) {
             writeZLong(cacheId);
@@ -226,6 +224,16 @@ public void writeCachedString(String string) throws IOException {
         writeString(string);
     }
 
+    @Override
+    public void writeOptionalCachedString(String str) throws IOException {
+        if (str == null) {
+            writeBoolean(false);
+        } else {
+            writeBoolean(true);
+            writeCachedString(str);
+        }
+    }
+
     private Integer esFieldIdFromCache(EsField field) {
         return cachedEsFields.get(field);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index bfbf5a8f0c66f..a1da269f896da 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -53,6 +53,7 @@
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.SkipQueryOnEmptyMappings;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.SkipQueryOnLimitZero;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.SplitInWithFoldableValue;
+import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteFilteredExpression;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSpatialSurrogates;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSurrogates;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.TranslateMetricsAggregate;
@@ -122,6 +123,9 @@ protected static Batch<LogicalPlan> substitutions() {
             "Substitutions",
             Limiter.ONCE,
             new ReplaceLookupWithJoin(),
+            // translate filtered expressions into aggregate with filters - can't use surrogate expressions because it was
+            // retrofitted for constant folding - this needs to be fixed
+            new SubstituteFilteredExpression(),
             new RemoveStatsOverride(),
             // first extract nested expressions inside aggs
             new ReplaceStatsNestedExpressionWithEval(),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java
index 0561865213a1b..0f08cd66444a3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
 
 public class FoldNull extends OptimizerRules.OptimizerExpressionRule<Expression> {
@@ -23,6 +24,16 @@ public FoldNull() {
     @Override
     public Expression rule(Expression e) {
         Expression result = tryReplaceIsNullIsNotNull(e);
+
+        // convert an aggregate null filter into a false
+        // perform this early to prevent the rule from converting the null filter into nullifying the whole expression
+        // P.S. this could be done inside the Aggregate but this place better centralizes the logic
+        if (e instanceof AggregateFunction agg) {
+            if (Expressions.isNull(agg.filter())) {
+                return agg.withFilter(Literal.of(agg.filter(), false));
+            }
+        }
+
         if (result != e) {
             return result;
         } else if (e instanceof In in) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java
index ed09d0bc16754..15e49c22a44db 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java
@@ -15,8 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
-import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
-import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
@@ -38,18 +36,13 @@ protected LogicalPlan rule(Filter filter) {
         LogicalPlan child = filter.child();
         Expression condition = filter.condition();
 
+        // TODO: Push down past STATS if the filter is only on the groups; but take into account how `STATS ... BY field` handles
+        // multi-values: It seems to be equivalent to `EVAL field = MV_DEDUPE(field) | MV_EXPAND(field) | STATS ... BY field`, where the
+        // last `STATS ... BY field` can assume that `field` is single-valued (to be checked more thoroughly).
+        // https://github.com/elastic/elasticsearch/issues/115311
         if (child instanceof Filter f) {
             // combine nodes into a single Filter with updated ANDed condition
             plan = f.with(Predicates.combineAnd(List.of(f.condition(), condition)));
-        } else if (child instanceof Aggregate agg) { // TODO: re-evaluate along with multi-value support
-            // Only push [parts of] a filter past an agg if these/it operates on agg's grouping[s], not output.
-            plan = maybePushDownPastUnary(
-                filter,
-                agg,
-                e -> e instanceof Attribute && agg.output().contains(e) && agg.groupings().contains(e) == false
-                    || e instanceof AggregateFunction,
-                NO_OP
-            );
         } else if (child instanceof Eval eval) {
             // Don't push if Filter (still) contains references to Eval's fields.
             // Account for simple aliases in the Eval, though - these shouldn't stop us.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java
index d74811518624a..559546d48eb7d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
@@ -25,8 +24,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.singleton;
-
 /**
  * Replace nested expressions over aggregates with synthetic eval post the aggregation
  * stats a = sum(a) + min(b) by x
@@ -71,16 +68,13 @@ protected LogicalPlan rule(Aggregate aggregate) {
 
         for (NamedExpression agg : aggs) {
             if (agg instanceof Alias as) {
-                // if the child a nested expression
+                // use intermediate variable to mark child as final for lambda use
                 Expression child = as.child();
 
                 // common case - handle duplicates
                 if (child instanceof AggregateFunction af) {
-                    AggregateFunction canonical = (AggregateFunction) af.canonical();
-                    Expression field = canonical.field().transformUp(e -> aliases.resolve(e, e));
-                    canonical = (AggregateFunction) canonical.replaceChildren(
-                        CollectionUtils.combine(singleton(field), canonical.parameters())
-                    );
+                    // canonical representation, with resolved aliases
+                    AggregateFunction canonical = (AggregateFunction) af.canonical().transformUp(e -> aliases.resolve(e, e));
 
                     Alias found = rootAggs.get(canonical);
                     // aggregate is new
@@ -130,7 +124,7 @@ protected LogicalPlan rule(Aggregate aggregate) {
         LogicalPlan plan = aggregate;
         if (changed.get()) {
             Source source = aggregate.source();
-            plan = new Aggregate(source, aggregate.child(), aggregate.aggregateType(), aggregate.groupings(), newAggs);
+            plan = aggregate.with(aggregate.child(), aggregate.groupings(), newAggs);
             if (newEvals.size() > 0) {
                 plan = new Eval(source, plan, newEvals);
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java
new file mode 100644
index 0000000000000..c8369d2b08a34
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
+import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.OptimizerExpressionRule;
+import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection;
+
+/**
+ * This rule should not be needed - the substitute infrastructure should be enough.
+ */
+public class SubstituteFilteredExpression extends OptimizerExpressionRule<FilteredExpression> {
+    public SubstituteFilteredExpression() {
+        super(TransformDirection.UP);
+    }
+
+    @Override
+    protected Expression rule(FilteredExpression filteredExpression) {
+        return filteredExpression.surrogate();
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
index be6e124502ba5..cde305e52a705 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
@@ -95,7 +95,7 @@ private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec) {
             }
             return comparison;
         });
-        if (rewritten.equals(filterExec.condition()) == false) {
+        if (rewritten.equals(filterExec.condition()) == false && canPushToSource(rewritten, x -> false)) {
             return new FilterExec(filterExec.source(), esQueryExec, rewritten);
         }
         return filterExec;
@@ -216,31 +216,40 @@ private Expression rewriteDistanceFilter(
         Number number,
         ComparisonType comparisonType
     ) {
+        DataType shapeDataType = getShapeDataType(spatialExp);
         Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(literalExp);
         if (geometry instanceof Point point) {
             double distance = number.doubleValue();
             Source source = comparison.source();
             if (comparisonType.lt) {
                 distance = comparisonType.eq ? distance : Math.nextDown(distance);
-                return new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp));
+                return new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp, shapeDataType));
             } else if (comparisonType.gt) {
                 distance = comparisonType.eq ? distance : Math.nextUp(distance);
-                return new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, distance, literalExp));
+                return new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, distance, literalExp, shapeDataType));
             } else if (comparisonType.eq) {
                 return new And(
                     source,
-                    new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp)),
-                    new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, Math.nextDown(distance), literalExp))
+                    new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp, shapeDataType)),
+                    new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, Math.nextDown(distance), literalExp, shapeDataType))
                 );
             }
         }
         return comparison;
     }
 
-    private Literal makeCircleLiteral(Point point, double distance, Expression literalExpression) {
+    private Literal makeCircleLiteral(Point point, double distance, Expression literalExpression, DataType shapeDataType) {
         var circle = new Circle(point.getX(), point.getY(), distance);
         var wkb = WellKnownBinary.toWKB(circle, ByteOrder.LITTLE_ENDIAN);
-        return new Literal(literalExpression.source(), new BytesRef(wkb), DataType.GEO_SHAPE);
+        return new Literal(literalExpression.source(), new BytesRef(wkb), shapeDataType);
+    }
+
+    private DataType getShapeDataType(Expression expression) {
+        return switch (expression.dataType()) {
+            case GEO_POINT, GEO_SHAPE -> DataType.GEO_SHAPE;
+            case CARTESIAN_POINT, CARTESIAN_SHAPE -> DataType.CARTESIAN_SHAPE;
+            default -> throw new IllegalArgumentException("Unsupported spatial data type: " + expression.dataType());
+        };
     }
 
     /**
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java
index 1ba966e318219..2209dffe5af06 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java
@@ -32,7 +32,8 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Queries;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
@@ -250,8 +251,10 @@ public static boolean canPushToSource(Expression exp, Predicate<FieldAttribute>
             return mqp.field() instanceof FieldAttribute && DataType.isString(mqp.field().dataType());
         } else if (exp instanceof StringQueryPredicate) {
             return true;
-        } else if (exp instanceof FullTextFunction) {
+        } else if (exp instanceof QueryString) {
             return true;
+        } else if (exp instanceof Match mf) {
+            return mf.field() instanceof FieldAttribute && DataType.isString(mf.field().dataType());
         }
         return false;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java
index 6db35fa0a06d6..855faf9df5ed2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.EvalExec;
-import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
 
@@ -82,17 +81,6 @@ public PhysicalPlan rewrite(TopNExec topNExec) {
         }
     }
 
-    /**
-     * TODO: Consider deleting this case entirely. We do not know if this is ever hit.
-     */
-    record PushableExchangeExec(ExchangeExec exchangeExec, EsQueryExec queryExec) implements Pushable {
-        public PhysicalPlan rewrite(TopNExec topNExec) {
-            var sorts = buildFieldSorts(topNExec.order());
-            var limit = topNExec.limit();
-            return exchangeExec.replaceChild(queryExec.withSorts(sorts).withLimit(limit));
-        }
-    }
-
     record PushableQueryExec(EsQueryExec queryExec) implements Pushable {
         public PhysicalPlan rewrite(TopNExec topNExec) {
             var sorts = buildFieldSorts(topNExec.order());
@@ -141,13 +129,6 @@ && canPushDownOrders(topNExec.order(), hasIdenticalDelegate)) {
             // With the simplest case of `FROM index | SORT ...` we only allow pushing down if the sort is on a field
             return new PushableQueryExec(queryExec);
         }
-        if (child instanceof ExchangeExec exchangeExec
-            && exchangeExec.child() instanceof EsQueryExec queryExec
-            && queryExec.canPushSorts()
-            && canPushDownOrders(topNExec.order(), hasIdenticalDelegate)) {
-            // When we have an exchange between the FROM and the SORT, we also only allow pushing down if the sort is on a field
-            return new PushableExchangeExec(exchangeExec, queryExec);
-        }
         if (child instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec queryExec && queryExec.canPushSorts()) {
             // When we have an EVAL between the FROM and the SORT, we consider pushing down if the sort is on a field and/or
             // a distance function defined in the EVAL. We also move the EVAL to after the SORT.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
index b5ca44826c051..2566da379af73 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
@@ -23,7 +23,6 @@ null
 null
 null
 null
-null
 '|'
 null
 null
@@ -63,6 +62,7 @@ null
 '*'
 '/'
 '%'
+'match'
 null
 null
 ']'
@@ -141,7 +141,6 @@ STATS
 WHERE
 DEV_INLINESTATS
 DEV_LOOKUP
-DEV_MATCH
 DEV_METRICS
 UNKNOWN_CMD
 LINE_COMMENT
@@ -186,6 +185,7 @@ MINUS
 ASTERISK
 SLASH
 PERCENT
+MATCH
 NAMED_OR_POSITIONAL_PARAM
 OPENING_BRACKET
 CLOSING_BRACKET
@@ -263,7 +263,6 @@ STATS
 WHERE
 DEV_INLINESTATS
 DEV_LOOKUP
-DEV_MATCH
 DEV_METRICS
 UNKNOWN_CMD
 LINE_COMMENT
@@ -318,7 +317,8 @@ MINUS
 ASTERISK
 SLASH
 PERCENT
-DEV_MATCH_OP
+MATCH
+NESTED_WHERE
 NAMED_OR_POSITIONAL_PARAM
 OPENING_BRACKET
 CLOSING_BRACKET
@@ -466,4 +466,4 @@ METRICS_MODE
 CLOSING_METRICS_MODE
 
 atn:
-[4, 0, 120, 1475, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 587, 8, 20, 11, 20, 12, 20, 588, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 597, 8, 21, 10, 21, 12, 21, 600, 9, 21, 1, 21, 3, 21, 603, 8, 21, 1, 21, 3, 21, 606, 8, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 615, 8, 22, 10, 22, 12, 22, 618, 9, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 4, 23, 626, 8, 23, 11, 23, 12, 23, 627, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 3, 29, 647, 8, 29, 1, 29, 4, 29, 650, 8, 29, 11, 29, 12, 29, 651, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 32, 3, 32, 661, 8, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 3, 34, 668, 8, 34, 1, 35, 1, 35, 1, 35, 5, 35, 673, 8, 35, 10, 35, 12, 35, 676, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 684, 8, 35, 10, 35, 12, 35, 687, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 3, 35, 694, 8, 35, 1, 35, 3, 35, 697, 8, 35, 3, 35, 699, 8, 35, 1, 36, 4, 36, 702, 8, 36, 11, 36, 12, 36, 703, 1, 37, 4, 37, 707, 8, 37, 11, 37, 12, 37, 708, 1, 37, 1, 37, 5, 37, 713, 8, 37, 10, 37, 12, 37, 716, 9, 37, 1, 37, 1, 37, 4, 37, 720, 8, 37, 11, 37, 12, 37, 721, 1, 37, 4, 37, 725, 8, 37, 11, 37, 12, 37, 726, 1, 37, 1, 37, 5, 37, 731, 8, 37, 10, 37, 12, 37, 734, 9, 37, 3, 37, 736, 8, 37, 1, 37, 1, 37, 1, 37, 1, 37, 4, 37, 742, 8, 37, 11, 37, 12, 37, 743, 1, 37, 1, 37, 3, 37, 748, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 875, 8, 74, 1, 74, 5, 74, 878, 8, 74, 10, 74, 12, 74, 881, 9, 74, 1, 74, 1, 74, 4, 74, 885, 8, 74, 11, 74, 12, 74, 886, 3, 74, 889, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 903, 8, 77, 10, 77, 12, 77, 906, 9, 77, 1, 77, 1, 77, 3, 77, 910, 8, 77, 1, 77, 4, 77, 913, 8, 77, 11, 77, 12, 77, 914, 3, 77, 917, 8, 77, 1, 78, 1, 78, 4, 78, 921, 8, 78, 11, 78, 12, 78, 922, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 1000, 8, 95, 1, 96, 4, 96, 1003, 8, 96, 11, 96, 12, 96, 1004, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 3, 107, 1052, 8, 107, 1, 108, 1, 108, 3, 108, 1056, 8, 108, 1, 108, 5, 108, 1059, 8, 108, 10, 108, 12, 108, 1062, 9, 108, 1, 108, 1, 108, 3, 108, 1066, 8, 108, 1, 108, 4, 108, 1069, 8, 108, 11, 108, 12, 108, 1070, 3, 108, 1073, 8, 108, 1, 109, 1, 109, 4, 109, 1077, 8, 109, 11, 109, 12, 109, 1078, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 129, 4, 129, 1162, 8, 129, 11, 129, 12, 129, 1163, 1, 129, 1, 129, 3, 129, 1168, 8, 129, 1, 129, 4, 129, 1171, 8, 129, 11, 129, 12, 129, 1172, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 4, 162, 1312, 8, 162, 11, 162, 12, 162, 1313, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 2, 616, 685, 0, 198, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 25, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 0, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 0, 233, 80, 235, 81, 237, 82, 239, 83, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 84, 255, 0, 257, 85, 259, 86, 261, 87, 263, 0, 265, 0, 267, 88, 269, 89, 271, 0, 273, 90, 275, 0, 277, 91, 279, 92, 281, 93, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 0, 301, 94, 303, 95, 305, 96, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 97, 321, 98, 323, 99, 325, 0, 327, 100, 329, 101, 331, 102, 333, 103, 335, 0, 337, 104, 339, 105, 341, 106, 343, 107, 345, 108, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 0, 361, 109, 363, 110, 365, 111, 367, 0, 369, 0, 371, 0, 373, 0, 375, 112, 377, 113, 379, 114, 381, 0, 383, 0, 385, 0, 387, 115, 389, 116, 391, 117, 393, 0, 395, 0, 397, 118, 399, 119, 401, 120, 403, 0, 405, 0, 407, 0, 409, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1503, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 1, 63, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 227, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 9, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 10, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 12, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 13, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 14, 409, 1, 0, 0, 0, 15, 411, 1, 0, 0, 0, 17, 421, 1, 0, 0, 0, 19, 428, 1, 0, 0, 0, 21, 437, 1, 0, 0, 0, 23, 444, 1, 0, 0, 0, 25, 454, 1, 0, 0, 0, 27, 461, 1, 0, 0, 0, 29, 468, 1, 0, 0, 0, 31, 475, 1, 0, 0, 0, 33, 483, 1, 0, 0, 0, 35, 495, 1, 0, 0, 0, 37, 504, 1, 0, 0, 0, 39, 510, 1, 0, 0, 0, 41, 517, 1, 0, 0, 0, 43, 524, 1, 0, 0, 0, 45, 532, 1, 0, 0, 0, 47, 540, 1, 0, 0, 0, 49, 555, 1, 0, 0, 0, 51, 565, 1, 0, 0, 0, 53, 574, 1, 0, 0, 0, 55, 586, 1, 0, 0, 0, 57, 592, 1, 0, 0, 0, 59, 609, 1, 0, 0, 0, 61, 625, 1, 0, 0, 0, 63, 631, 1, 0, 0, 0, 65, 635, 1, 0, 0, 0, 67, 637, 1, 0, 0, 0, 69, 639, 1, 0, 0, 0, 71, 642, 1, 0, 0, 0, 73, 644, 1, 0, 0, 0, 75, 653, 1, 0, 0, 0, 77, 655, 1, 0, 0, 0, 79, 660, 1, 0, 0, 0, 81, 662, 1, 0, 0, 0, 83, 667, 1, 0, 0, 0, 85, 698, 1, 0, 0, 0, 87, 701, 1, 0, 0, 0, 89, 747, 1, 0, 0, 0, 91, 749, 1, 0, 0, 0, 93, 752, 1, 0, 0, 0, 95, 756, 1, 0, 0, 0, 97, 760, 1, 0, 0, 0, 99, 762, 1, 0, 0, 0, 101, 765, 1, 0, 0, 0, 103, 767, 1, 0, 0, 0, 105, 772, 1, 0, 0, 0, 107, 774, 1, 0, 0, 0, 109, 780, 1, 0, 0, 0, 111, 786, 1, 0, 0, 0, 113, 789, 1, 0, 0, 0, 115, 792, 1, 0, 0, 0, 117, 797, 1, 0, 0, 0, 119, 802, 1, 0, 0, 0, 121, 804, 1, 0, 0, 0, 123, 808, 1, 0, 0, 0, 125, 813, 1, 0, 0, 0, 127, 819, 1, 0, 0, 0, 129, 822, 1, 0, 0, 0, 131, 824, 1, 0, 0, 0, 133, 830, 1, 0, 0, 0, 135, 832, 1, 0, 0, 0, 137, 837, 1, 0, 0, 0, 139, 840, 1, 0, 0, 0, 141, 843, 1, 0, 0, 0, 143, 846, 1, 0, 0, 0, 145, 848, 1, 0, 0, 0, 147, 851, 1, 0, 0, 0, 149, 853, 1, 0, 0, 0, 151, 856, 1, 0, 0, 0, 153, 858, 1, 0, 0, 0, 155, 860, 1, 0, 0, 0, 157, 862, 1, 0, 0, 0, 159, 864, 1, 0, 0, 0, 161, 866, 1, 0, 0, 0, 163, 888, 1, 0, 0, 0, 165, 890, 1, 0, 0, 0, 167, 895, 1, 0, 0, 0, 169, 916, 1, 0, 0, 0, 171, 918, 1, 0, 0, 0, 173, 926, 1, 0, 0, 0, 175, 928, 1, 0, 0, 0, 177, 932, 1, 0, 0, 0, 179, 936, 1, 0, 0, 0, 181, 940, 1, 0, 0, 0, 183, 945, 1, 0, 0, 0, 185, 950, 1, 0, 0, 0, 187, 954, 1, 0, 0, 0, 189, 958, 1, 0, 0, 0, 191, 962, 1, 0, 0, 0, 193, 967, 1, 0, 0, 0, 195, 971, 1, 0, 0, 0, 197, 975, 1, 0, 0, 0, 199, 979, 1, 0, 0, 0, 201, 983, 1, 0, 0, 0, 203, 987, 1, 0, 0, 0, 205, 999, 1, 0, 0, 0, 207, 1002, 1, 0, 0, 0, 209, 1006, 1, 0, 0, 0, 211, 1010, 1, 0, 0, 0, 213, 1014, 1, 0, 0, 0, 215, 1018, 1, 0, 0, 0, 217, 1022, 1, 0, 0, 0, 219, 1026, 1, 0, 0, 0, 221, 1031, 1, 0, 0, 0, 223, 1035, 1, 0, 0, 0, 225, 1039, 1, 0, 0, 0, 227, 1043, 1, 0, 0, 0, 229, 1051, 1, 0, 0, 0, 231, 1072, 1, 0, 0, 0, 233, 1076, 1, 0, 0, 0, 235, 1080, 1, 0, 0, 0, 237, 1084, 1, 0, 0, 0, 239, 1088, 1, 0, 0, 0, 241, 1092, 1, 0, 0, 0, 243, 1097, 1, 0, 0, 0, 245, 1101, 1, 0, 0, 0, 247, 1105, 1, 0, 0, 0, 249, 1109, 1, 0, 0, 0, 251, 1113, 1, 0, 0, 0, 253, 1117, 1, 0, 0, 0, 255, 1120, 1, 0, 0, 0, 257, 1124, 1, 0, 0, 0, 259, 1128, 1, 0, 0, 0, 261, 1132, 1, 0, 0, 0, 263, 1136, 1, 0, 0, 0, 265, 1141, 1, 0, 0, 0, 267, 1146, 1, 0, 0, 0, 269, 1151, 1, 0, 0, 0, 271, 1158, 1, 0, 0, 0, 273, 1167, 1, 0, 0, 0, 275, 1174, 1, 0, 0, 0, 277, 1178, 1, 0, 0, 0, 279, 1182, 1, 0, 0, 0, 281, 1186, 1, 0, 0, 0, 283, 1190, 1, 0, 0, 0, 285, 1196, 1, 0, 0, 0, 287, 1200, 1, 0, 0, 0, 289, 1204, 1, 0, 0, 0, 291, 1208, 1, 0, 0, 0, 293, 1212, 1, 0, 0, 0, 295, 1216, 1, 0, 0, 0, 297, 1220, 1, 0, 0, 0, 299, 1224, 1, 0, 0, 0, 301, 1228, 1, 0, 0, 0, 303, 1232, 1, 0, 0, 0, 305, 1236, 1, 0, 0, 0, 307, 1240, 1, 0, 0, 0, 309, 1245, 1, 0, 0, 0, 311, 1249, 1, 0, 0, 0, 313, 1253, 1, 0, 0, 0, 315, 1257, 1, 0, 0, 0, 317, 1261, 1, 0, 0, 0, 319, 1265, 1, 0, 0, 0, 321, 1269, 1, 0, 0, 0, 323, 1273, 1, 0, 0, 0, 325, 1277, 1, 0, 0, 0, 327, 1282, 1, 0, 0, 0, 329, 1287, 1, 0, 0, 0, 331, 1291, 1, 0, 0, 0, 333, 1295, 1, 0, 0, 0, 335, 1299, 1, 0, 0, 0, 337, 1304, 1, 0, 0, 0, 339, 1311, 1, 0, 0, 0, 341, 1315, 1, 0, 0, 0, 343, 1319, 1, 0, 0, 0, 345, 1323, 1, 0, 0, 0, 347, 1327, 1, 0, 0, 0, 349, 1332, 1, 0, 0, 0, 351, 1336, 1, 0, 0, 0, 353, 1340, 1, 0, 0, 0, 355, 1344, 1, 0, 0, 0, 357, 1349, 1, 0, 0, 0, 359, 1353, 1, 0, 0, 0, 361, 1357, 1, 0, 0, 0, 363, 1361, 1, 0, 0, 0, 365, 1365, 1, 0, 0, 0, 367, 1369, 1, 0, 0, 0, 369, 1375, 1, 0, 0, 0, 371, 1379, 1, 0, 0, 0, 373, 1383, 1, 0, 0, 0, 375, 1387, 1, 0, 0, 0, 377, 1391, 1, 0, 0, 0, 379, 1395, 1, 0, 0, 0, 381, 1399, 1, 0, 0, 0, 383, 1404, 1, 0, 0, 0, 385, 1410, 1, 0, 0, 0, 387, 1416, 1, 0, 0, 0, 389, 1420, 1, 0, 0, 0, 391, 1424, 1, 0, 0, 0, 393, 1428, 1, 0, 0, 0, 395, 1434, 1, 0, 0, 0, 397, 1440, 1, 0, 0, 0, 399, 1444, 1, 0, 0, 0, 401, 1448, 1, 0, 0, 0, 403, 1452, 1, 0, 0, 0, 405, 1458, 1, 0, 0, 0, 407, 1464, 1, 0, 0, 0, 409, 1470, 1, 0, 0, 0, 411, 412, 7, 0, 0, 0, 412, 413, 7, 1, 0, 0, 413, 414, 7, 2, 0, 0, 414, 415, 7, 2, 0, 0, 415, 416, 7, 3, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 5, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 0, 0, 0, 420, 16, 1, 0, 0, 0, 421, 422, 7, 0, 0, 0, 422, 423, 7, 6, 0, 0, 423, 424, 7, 7, 0, 0, 424, 425, 7, 8, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 1, 1, 0, 427, 18, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 9, 0, 0, 430, 431, 7, 6, 0, 0, 431, 432, 7, 1, 0, 0, 432, 433, 7, 4, 0, 0, 433, 434, 7, 10, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 2, 2, 0, 436, 20, 1, 0, 0, 0, 437, 438, 7, 3, 0, 0, 438, 439, 7, 11, 0, 0, 439, 440, 7, 12, 0, 0, 440, 441, 7, 13, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 3, 0, 0, 443, 22, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 446, 7, 14, 0, 0, 446, 447, 7, 8, 0, 0, 447, 448, 7, 13, 0, 0, 448, 449, 7, 12, 0, 0, 449, 450, 7, 1, 0, 0, 450, 451, 7, 9, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 4, 3, 0, 453, 24, 1, 0, 0, 0, 454, 455, 7, 15, 0, 0, 455, 456, 7, 6, 0, 0, 456, 457, 7, 7, 0, 0, 457, 458, 7, 16, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 5, 4, 0, 460, 26, 1, 0, 0, 0, 461, 462, 7, 17, 0, 0, 462, 463, 7, 6, 0, 0, 463, 464, 7, 7, 0, 0, 464, 465, 7, 18, 0, 0, 465, 466, 1, 0, 0, 0, 466, 467, 6, 6, 0, 0, 467, 28, 1, 0, 0, 0, 468, 469, 7, 18, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 8, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 1, 0, 474, 30, 1, 0, 0, 0, 475, 476, 7, 13, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 16, 0, 0, 478, 479, 7, 1, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 8, 0, 0, 482, 32, 1, 0, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 11, 0, 0, 485, 486, 5, 95, 0, 0, 486, 487, 7, 3, 0, 0, 487, 488, 7, 14, 0, 0, 488, 489, 7, 8, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 9, 0, 0, 491, 492, 7, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 9, 5, 0, 494, 34, 1, 0, 0, 0, 495, 496, 7, 6, 0, 0, 496, 497, 7, 3, 0, 0, 497, 498, 7, 9, 0, 0, 498, 499, 7, 12, 0, 0, 499, 500, 7, 16, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 10, 6, 0, 503, 36, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 506, 7, 7, 0, 0, 506, 507, 7, 19, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 11, 0, 0, 509, 38, 1, 0, 0, 0, 510, 511, 7, 2, 0, 0, 511, 512, 7, 10, 0, 0, 512, 513, 7, 7, 0, 0, 513, 514, 7, 19, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 12, 7, 0, 516, 40, 1, 0, 0, 0, 517, 518, 7, 2, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 13, 0, 0, 523, 42, 1, 0, 0, 0, 524, 525, 7, 2, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 12, 0, 0, 527, 528, 7, 5, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 14, 0, 0, 531, 44, 1, 0, 0, 0, 532, 533, 7, 19, 0, 0, 533, 534, 7, 10, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 7, 6, 0, 0, 536, 537, 7, 3, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 15, 0, 0, 539, 46, 1, 0, 0, 0, 540, 541, 4, 16, 0, 0, 541, 542, 7, 1, 0, 0, 542, 543, 7, 9, 0, 0, 543, 544, 7, 13, 0, 0, 544, 545, 7, 1, 0, 0, 545, 546, 7, 9, 0, 0, 546, 547, 7, 3, 0, 0, 547, 548, 7, 2, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 12, 0, 0, 550, 551, 7, 5, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 16, 0, 0, 554, 48, 1, 0, 0, 0, 555, 556, 4, 17, 1, 0, 556, 557, 7, 13, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 7, 0, 0, 559, 560, 7, 18, 0, 0, 560, 561, 7, 20, 0, 0, 561, 562, 7, 8, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 17, 8, 0, 564, 50, 1, 0, 0, 0, 565, 566, 4, 18, 2, 0, 566, 567, 7, 16, 0, 0, 567, 568, 7, 12, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 4, 0, 0, 570, 571, 7, 10, 0, 0, 571, 572, 1, 0, 0, 0, 572, 573, 6, 18, 0, 0, 573, 52, 1, 0, 0, 0, 574, 575, 4, 19, 3, 0, 575, 576, 7, 16, 0, 0, 576, 577, 7, 3, 0, 0, 577, 578, 7, 5, 0, 0, 578, 579, 7, 6, 0, 0, 579, 580, 7, 1, 0, 0, 580, 581, 7, 4, 0, 0, 581, 582, 7, 2, 0, 0, 582, 583, 1, 0, 0, 0, 583, 584, 6, 19, 9, 0, 584, 54, 1, 0, 0, 0, 585, 587, 8, 21, 0, 0, 586, 585, 1, 0, 0, 0, 587, 588, 1, 0, 0, 0, 588, 586, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 6, 20, 0, 0, 591, 56, 1, 0, 0, 0, 592, 593, 5, 47, 0, 0, 593, 594, 5, 47, 0, 0, 594, 598, 1, 0, 0, 0, 595, 597, 8, 22, 0, 0, 596, 595, 1, 0, 0, 0, 597, 600, 1, 0, 0, 0, 598, 596, 1, 0, 0, 0, 598, 599, 1, 0, 0, 0, 599, 602, 1, 0, 0, 0, 600, 598, 1, 0, 0, 0, 601, 603, 5, 13, 0, 0, 602, 601, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 605, 1, 0, 0, 0, 604, 606, 5, 10, 0, 0, 605, 604, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 608, 6, 21, 10, 0, 608, 58, 1, 0, 0, 0, 609, 610, 5, 47, 0, 0, 610, 611, 5, 42, 0, 0, 611, 616, 1, 0, 0, 0, 612, 615, 3, 59, 22, 0, 613, 615, 9, 0, 0, 0, 614, 612, 1, 0, 0, 0, 614, 613, 1, 0, 0, 0, 615, 618, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 616, 614, 1, 0, 0, 0, 617, 619, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 619, 620, 5, 42, 0, 0, 620, 621, 5, 47, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 6, 22, 10, 0, 623, 60, 1, 0, 0, 0, 624, 626, 7, 23, 0, 0, 625, 624, 1, 0, 0, 0, 626, 627, 1, 0, 0, 0, 627, 625, 1, 0, 0, 0, 627, 628, 1, 0, 0, 0, 628, 629, 1, 0, 0, 0, 629, 630, 6, 23, 10, 0, 630, 62, 1, 0, 0, 0, 631, 632, 5, 124, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 6, 24, 11, 0, 634, 64, 1, 0, 0, 0, 635, 636, 7, 24, 0, 0, 636, 66, 1, 0, 0, 0, 637, 638, 7, 25, 0, 0, 638, 68, 1, 0, 0, 0, 639, 640, 5, 92, 0, 0, 640, 641, 7, 26, 0, 0, 641, 70, 1, 0, 0, 0, 642, 643, 8, 27, 0, 0, 643, 72, 1, 0, 0, 0, 644, 646, 7, 3, 0, 0, 645, 647, 7, 28, 0, 0, 646, 645, 1, 0, 0, 0, 646, 647, 1, 0, 0, 0, 647, 649, 1, 0, 0, 0, 648, 650, 3, 65, 25, 0, 649, 648, 1, 0, 0, 0, 650, 651, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 651, 652, 1, 0, 0, 0, 652, 74, 1, 0, 0, 0, 653, 654, 5, 64, 0, 0, 654, 76, 1, 0, 0, 0, 655, 656, 5, 96, 0, 0, 656, 78, 1, 0, 0, 0, 657, 661, 8, 29, 0, 0, 658, 659, 5, 96, 0, 0, 659, 661, 5, 96, 0, 0, 660, 657, 1, 0, 0, 0, 660, 658, 1, 0, 0, 0, 661, 80, 1, 0, 0, 0, 662, 663, 5, 95, 0, 0, 663, 82, 1, 0, 0, 0, 664, 668, 3, 67, 26, 0, 665, 668, 3, 65, 25, 0, 666, 668, 3, 81, 33, 0, 667, 664, 1, 0, 0, 0, 667, 665, 1, 0, 0, 0, 667, 666, 1, 0, 0, 0, 668, 84, 1, 0, 0, 0, 669, 674, 5, 34, 0, 0, 670, 673, 3, 69, 27, 0, 671, 673, 3, 71, 28, 0, 672, 670, 1, 0, 0, 0, 672, 671, 1, 0, 0, 0, 673, 676, 1, 0, 0, 0, 674, 672, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 675, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 699, 5, 34, 0, 0, 678, 679, 5, 34, 0, 0, 679, 680, 5, 34, 0, 0, 680, 681, 5, 34, 0, 0, 681, 685, 1, 0, 0, 0, 682, 684, 8, 22, 0, 0, 683, 682, 1, 0, 0, 0, 684, 687, 1, 0, 0, 0, 685, 686, 1, 0, 0, 0, 685, 683, 1, 0, 0, 0, 686, 688, 1, 0, 0, 0, 687, 685, 1, 0, 0, 0, 688, 689, 5, 34, 0, 0, 689, 690, 5, 34, 0, 0, 690, 691, 5, 34, 0, 0, 691, 693, 1, 0, 0, 0, 692, 694, 5, 34, 0, 0, 693, 692, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 696, 1, 0, 0, 0, 695, 697, 5, 34, 0, 0, 696, 695, 1, 0, 0, 0, 696, 697, 1, 0, 0, 0, 697, 699, 1, 0, 0, 0, 698, 669, 1, 0, 0, 0, 698, 678, 1, 0, 0, 0, 699, 86, 1, 0, 0, 0, 700, 702, 3, 65, 25, 0, 701, 700, 1, 0, 0, 0, 702, 703, 1, 0, 0, 0, 703, 701, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 88, 1, 0, 0, 0, 705, 707, 3, 65, 25, 0, 706, 705, 1, 0, 0, 0, 707, 708, 1, 0, 0, 0, 708, 706, 1, 0, 0, 0, 708, 709, 1, 0, 0, 0, 709, 710, 1, 0, 0, 0, 710, 714, 3, 105, 45, 0, 711, 713, 3, 65, 25, 0, 712, 711, 1, 0, 0, 0, 713, 716, 1, 0, 0, 0, 714, 712, 1, 0, 0, 0, 714, 715, 1, 0, 0, 0, 715, 748, 1, 0, 0, 0, 716, 714, 1, 0, 0, 0, 717, 719, 3, 105, 45, 0, 718, 720, 3, 65, 25, 0, 719, 718, 1, 0, 0, 0, 720, 721, 1, 0, 0, 0, 721, 719, 1, 0, 0, 0, 721, 722, 1, 0, 0, 0, 722, 748, 1, 0, 0, 0, 723, 725, 3, 65, 25, 0, 724, 723, 1, 0, 0, 0, 725, 726, 1, 0, 0, 0, 726, 724, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 735, 1, 0, 0, 0, 728, 732, 3, 105, 45, 0, 729, 731, 3, 65, 25, 0, 730, 729, 1, 0, 0, 0, 731, 734, 1, 0, 0, 0, 732, 730, 1, 0, 0, 0, 732, 733, 1, 0, 0, 0, 733, 736, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 735, 728, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 738, 3, 73, 29, 0, 738, 748, 1, 0, 0, 0, 739, 741, 3, 105, 45, 0, 740, 742, 3, 65, 25, 0, 741, 740, 1, 0, 0, 0, 742, 743, 1, 0, 0, 0, 743, 741, 1, 0, 0, 0, 743, 744, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 746, 3, 73, 29, 0, 746, 748, 1, 0, 0, 0, 747, 706, 1, 0, 0, 0, 747, 717, 1, 0, 0, 0, 747, 724, 1, 0, 0, 0, 747, 739, 1, 0, 0, 0, 748, 90, 1, 0, 0, 0, 749, 750, 7, 30, 0, 0, 750, 751, 7, 31, 0, 0, 751, 92, 1, 0, 0, 0, 752, 753, 7, 12, 0, 0, 753, 754, 7, 9, 0, 0, 754, 755, 7, 0, 0, 0, 755, 94, 1, 0, 0, 0, 756, 757, 7, 12, 0, 0, 757, 758, 7, 2, 0, 0, 758, 759, 7, 4, 0, 0, 759, 96, 1, 0, 0, 0, 760, 761, 5, 61, 0, 0, 761, 98, 1, 0, 0, 0, 762, 763, 5, 58, 0, 0, 763, 764, 5, 58, 0, 0, 764, 100, 1, 0, 0, 0, 765, 766, 5, 44, 0, 0, 766, 102, 1, 0, 0, 0, 767, 768, 7, 0, 0, 0, 768, 769, 7, 3, 0, 0, 769, 770, 7, 2, 0, 0, 770, 771, 7, 4, 0, 0, 771, 104, 1, 0, 0, 0, 772, 773, 5, 46, 0, 0, 773, 106, 1, 0, 0, 0, 774, 775, 7, 15, 0, 0, 775, 776, 7, 12, 0, 0, 776, 777, 7, 13, 0, 0, 777, 778, 7, 2, 0, 0, 778, 779, 7, 3, 0, 0, 779, 108, 1, 0, 0, 0, 780, 781, 7, 15, 0, 0, 781, 782, 7, 1, 0, 0, 782, 783, 7, 6, 0, 0, 783, 784, 7, 2, 0, 0, 784, 785, 7, 5, 0, 0, 785, 110, 1, 0, 0, 0, 786, 787, 7, 1, 0, 0, 787, 788, 7, 9, 0, 0, 788, 112, 1, 0, 0, 0, 789, 790, 7, 1, 0, 0, 790, 791, 7, 2, 0, 0, 791, 114, 1, 0, 0, 0, 792, 793, 7, 13, 0, 0, 793, 794, 7, 12, 0, 0, 794, 795, 7, 2, 0, 0, 795, 796, 7, 5, 0, 0, 796, 116, 1, 0, 0, 0, 797, 798, 7, 13, 0, 0, 798, 799, 7, 1, 0, 0, 799, 800, 7, 18, 0, 0, 800, 801, 7, 3, 0, 0, 801, 118, 1, 0, 0, 0, 802, 803, 5, 40, 0, 0, 803, 120, 1, 0, 0, 0, 804, 805, 7, 9, 0, 0, 805, 806, 7, 7, 0, 0, 806, 807, 7, 5, 0, 0, 807, 122, 1, 0, 0, 0, 808, 809, 7, 9, 0, 0, 809, 810, 7, 20, 0, 0, 810, 811, 7, 13, 0, 0, 811, 812, 7, 13, 0, 0, 812, 124, 1, 0, 0, 0, 813, 814, 7, 9, 0, 0, 814, 815, 7, 20, 0, 0, 815, 816, 7, 13, 0, 0, 816, 817, 7, 13, 0, 0, 817, 818, 7, 2, 0, 0, 818, 126, 1, 0, 0, 0, 819, 820, 7, 7, 0, 0, 820, 821, 7, 6, 0, 0, 821, 128, 1, 0, 0, 0, 822, 823, 5, 63, 0, 0, 823, 130, 1, 0, 0, 0, 824, 825, 7, 6, 0, 0, 825, 826, 7, 13, 0, 0, 826, 827, 7, 1, 0, 0, 827, 828, 7, 18, 0, 0, 828, 829, 7, 3, 0, 0, 829, 132, 1, 0, 0, 0, 830, 831, 5, 41, 0, 0, 831, 134, 1, 0, 0, 0, 832, 833, 7, 5, 0, 0, 833, 834, 7, 6, 0, 0, 834, 835, 7, 20, 0, 0, 835, 836, 7, 3, 0, 0, 836, 136, 1, 0, 0, 0, 837, 838, 5, 61, 0, 0, 838, 839, 5, 61, 0, 0, 839, 138, 1, 0, 0, 0, 840, 841, 5, 61, 0, 0, 841, 842, 5, 126, 0, 0, 842, 140, 1, 0, 0, 0, 843, 844, 5, 33, 0, 0, 844, 845, 5, 61, 0, 0, 845, 142, 1, 0, 0, 0, 846, 847, 5, 60, 0, 0, 847, 144, 1, 0, 0, 0, 848, 849, 5, 60, 0, 0, 849, 850, 5, 61, 0, 0, 850, 146, 1, 0, 0, 0, 851, 852, 5, 62, 0, 0, 852, 148, 1, 0, 0, 0, 853, 854, 5, 62, 0, 0, 854, 855, 5, 61, 0, 0, 855, 150, 1, 0, 0, 0, 856, 857, 5, 43, 0, 0, 857, 152, 1, 0, 0, 0, 858, 859, 5, 45, 0, 0, 859, 154, 1, 0, 0, 0, 860, 861, 5, 42, 0, 0, 861, 156, 1, 0, 0, 0, 862, 863, 5, 47, 0, 0, 863, 158, 1, 0, 0, 0, 864, 865, 5, 37, 0, 0, 865, 160, 1, 0, 0, 0, 866, 867, 4, 73, 4, 0, 867, 868, 3, 51, 18, 0, 868, 869, 1, 0, 0, 0, 869, 870, 6, 73, 12, 0, 870, 162, 1, 0, 0, 0, 871, 874, 3, 129, 57, 0, 872, 875, 3, 67, 26, 0, 873, 875, 3, 81, 33, 0, 874, 872, 1, 0, 0, 0, 874, 873, 1, 0, 0, 0, 875, 879, 1, 0, 0, 0, 876, 878, 3, 83, 34, 0, 877, 876, 1, 0, 0, 0, 878, 881, 1, 0, 0, 0, 879, 877, 1, 0, 0, 0, 879, 880, 1, 0, 0, 0, 880, 889, 1, 0, 0, 0, 881, 879, 1, 0, 0, 0, 882, 884, 3, 129, 57, 0, 883, 885, 3, 65, 25, 0, 884, 883, 1, 0, 0, 0, 885, 886, 1, 0, 0, 0, 886, 884, 1, 0, 0, 0, 886, 887, 1, 0, 0, 0, 887, 889, 1, 0, 0, 0, 888, 871, 1, 0, 0, 0, 888, 882, 1, 0, 0, 0, 889, 164, 1, 0, 0, 0, 890, 891, 5, 91, 0, 0, 891, 892, 1, 0, 0, 0, 892, 893, 6, 75, 0, 0, 893, 894, 6, 75, 0, 0, 894, 166, 1, 0, 0, 0, 895, 896, 5, 93, 0, 0, 896, 897, 1, 0, 0, 0, 897, 898, 6, 76, 11, 0, 898, 899, 6, 76, 11, 0, 899, 168, 1, 0, 0, 0, 900, 904, 3, 67, 26, 0, 901, 903, 3, 83, 34, 0, 902, 901, 1, 0, 0, 0, 903, 906, 1, 0, 0, 0, 904, 902, 1, 0, 0, 0, 904, 905, 1, 0, 0, 0, 905, 917, 1, 0, 0, 0, 906, 904, 1, 0, 0, 0, 907, 910, 3, 81, 33, 0, 908, 910, 3, 75, 30, 0, 909, 907, 1, 0, 0, 0, 909, 908, 1, 0, 0, 0, 910, 912, 1, 0, 0, 0, 911, 913, 3, 83, 34, 0, 912, 911, 1, 0, 0, 0, 913, 914, 1, 0, 0, 0, 914, 912, 1, 0, 0, 0, 914, 915, 1, 0, 0, 0, 915, 917, 1, 0, 0, 0, 916, 900, 1, 0, 0, 0, 916, 909, 1, 0, 0, 0, 917, 170, 1, 0, 0, 0, 918, 920, 3, 77, 31, 0, 919, 921, 3, 79, 32, 0, 920, 919, 1, 0, 0, 0, 921, 922, 1, 0, 0, 0, 922, 920, 1, 0, 0, 0, 922, 923, 1, 0, 0, 0, 923, 924, 1, 0, 0, 0, 924, 925, 3, 77, 31, 0, 925, 172, 1, 0, 0, 0, 926, 927, 3, 171, 78, 0, 927, 174, 1, 0, 0, 0, 928, 929, 3, 57, 21, 0, 929, 930, 1, 0, 0, 0, 930, 931, 6, 80, 10, 0, 931, 176, 1, 0, 0, 0, 932, 933, 3, 59, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 81, 10, 0, 935, 178, 1, 0, 0, 0, 936, 937, 3, 61, 23, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 82, 10, 0, 939, 180, 1, 0, 0, 0, 940, 941, 3, 165, 75, 0, 941, 942, 1, 0, 0, 0, 942, 943, 6, 83, 13, 0, 943, 944, 6, 83, 14, 0, 944, 182, 1, 0, 0, 0, 945, 946, 3, 63, 24, 0, 946, 947, 1, 0, 0, 0, 947, 948, 6, 84, 15, 0, 948, 949, 6, 84, 11, 0, 949, 184, 1, 0, 0, 0, 950, 951, 3, 61, 23, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 85, 10, 0, 953, 186, 1, 0, 0, 0, 954, 955, 3, 57, 21, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 86, 10, 0, 957, 188, 1, 0, 0, 0, 958, 959, 3, 59, 22, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 87, 10, 0, 961, 190, 1, 0, 0, 0, 962, 963, 3, 63, 24, 0, 963, 964, 1, 0, 0, 0, 964, 965, 6, 88, 15, 0, 965, 966, 6, 88, 11, 0, 966, 192, 1, 0, 0, 0, 967, 968, 3, 165, 75, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 89, 13, 0, 970, 194, 1, 0, 0, 0, 971, 972, 3, 167, 76, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 90, 16, 0, 974, 196, 1, 0, 0, 0, 975, 976, 3, 337, 161, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 91, 17, 0, 978, 198, 1, 0, 0, 0, 979, 980, 3, 101, 43, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 92, 18, 0, 982, 200, 1, 0, 0, 0, 983, 984, 3, 97, 41, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 93, 19, 0, 986, 202, 1, 0, 0, 0, 987, 988, 7, 16, 0, 0, 988, 989, 7, 3, 0, 0, 989, 990, 7, 5, 0, 0, 990, 991, 7, 12, 0, 0, 991, 992, 7, 0, 0, 0, 992, 993, 7, 12, 0, 0, 993, 994, 7, 5, 0, 0, 994, 995, 7, 12, 0, 0, 995, 204, 1, 0, 0, 0, 996, 1000, 8, 32, 0, 0, 997, 998, 5, 47, 0, 0, 998, 1000, 8, 33, 0, 0, 999, 996, 1, 0, 0, 0, 999, 997, 1, 0, 0, 0, 1000, 206, 1, 0, 0, 0, 1001, 1003, 3, 205, 95, 0, 1002, 1001, 1, 0, 0, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1002, 1, 0, 0, 0, 1004, 1005, 1, 0, 0, 0, 1005, 208, 1, 0, 0, 0, 1006, 1007, 3, 207, 96, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 97, 20, 0, 1009, 210, 1, 0, 0, 0, 1010, 1011, 3, 85, 35, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 98, 21, 0, 1013, 212, 1, 0, 0, 0, 1014, 1015, 3, 57, 21, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 99, 10, 0, 1017, 214, 1, 0, 0, 0, 1018, 1019, 3, 59, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 100, 10, 0, 1021, 216, 1, 0, 0, 0, 1022, 1023, 3, 61, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 101, 10, 0, 1025, 218, 1, 0, 0, 0, 1026, 1027, 3, 63, 24, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 102, 15, 0, 1029, 1030, 6, 102, 11, 0, 1030, 220, 1, 0, 0, 0, 1031, 1032, 3, 105, 45, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1034, 6, 103, 22, 0, 1034, 222, 1, 0, 0, 0, 1035, 1036, 3, 101, 43, 0, 1036, 1037, 1, 0, 0, 0, 1037, 1038, 6, 104, 18, 0, 1038, 224, 1, 0, 0, 0, 1039, 1040, 3, 129, 57, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 105, 23, 0, 1042, 226, 1, 0, 0, 0, 1043, 1044, 3, 163, 74, 0, 1044, 1045, 1, 0, 0, 0, 1045, 1046, 6, 106, 24, 0, 1046, 228, 1, 0, 0, 0, 1047, 1052, 3, 67, 26, 0, 1048, 1052, 3, 65, 25, 0, 1049, 1052, 3, 81, 33, 0, 1050, 1052, 3, 155, 70, 0, 1051, 1047, 1, 0, 0, 0, 1051, 1048, 1, 0, 0, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1050, 1, 0, 0, 0, 1052, 230, 1, 0, 0, 0, 1053, 1056, 3, 67, 26, 0, 1054, 1056, 3, 155, 70, 0, 1055, 1053, 1, 0, 0, 0, 1055, 1054, 1, 0, 0, 0, 1056, 1060, 1, 0, 0, 0, 1057, 1059, 3, 229, 107, 0, 1058, 1057, 1, 0, 0, 0, 1059, 1062, 1, 0, 0, 0, 1060, 1058, 1, 0, 0, 0, 1060, 1061, 1, 0, 0, 0, 1061, 1073, 1, 0, 0, 0, 1062, 1060, 1, 0, 0, 0, 1063, 1066, 3, 81, 33, 0, 1064, 1066, 3, 75, 30, 0, 1065, 1063, 1, 0, 0, 0, 1065, 1064, 1, 0, 0, 0, 1066, 1068, 1, 0, 0, 0, 1067, 1069, 3, 229, 107, 0, 1068, 1067, 1, 0, 0, 0, 1069, 1070, 1, 0, 0, 0, 1070, 1068, 1, 0, 0, 0, 1070, 1071, 1, 0, 0, 0, 1071, 1073, 1, 0, 0, 0, 1072, 1055, 1, 0, 0, 0, 1072, 1065, 1, 0, 0, 0, 1073, 232, 1, 0, 0, 0, 1074, 1077, 3, 231, 108, 0, 1075, 1077, 3, 171, 78, 0, 1076, 1074, 1, 0, 0, 0, 1076, 1075, 1, 0, 0, 0, 1077, 1078, 1, 0, 0, 0, 1078, 1076, 1, 0, 0, 0, 1078, 1079, 1, 0, 0, 0, 1079, 234, 1, 0, 0, 0, 1080, 1081, 3, 57, 21, 0, 1081, 1082, 1, 0, 0, 0, 1082, 1083, 6, 110, 10, 0, 1083, 236, 1, 0, 0, 0, 1084, 1085, 3, 59, 22, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 6, 111, 10, 0, 1087, 238, 1, 0, 0, 0, 1088, 1089, 3, 61, 23, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 112, 10, 0, 1091, 240, 1, 0, 0, 0, 1092, 1093, 3, 63, 24, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1095, 6, 113, 15, 0, 1095, 1096, 6, 113, 11, 0, 1096, 242, 1, 0, 0, 0, 1097, 1098, 3, 97, 41, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 114, 19, 0, 1100, 244, 1, 0, 0, 0, 1101, 1102, 3, 101, 43, 0, 1102, 1103, 1, 0, 0, 0, 1103, 1104, 6, 115, 18, 0, 1104, 246, 1, 0, 0, 0, 1105, 1106, 3, 105, 45, 0, 1106, 1107, 1, 0, 0, 0, 1107, 1108, 6, 116, 22, 0, 1108, 248, 1, 0, 0, 0, 1109, 1110, 3, 129, 57, 0, 1110, 1111, 1, 0, 0, 0, 1111, 1112, 6, 117, 23, 0, 1112, 250, 1, 0, 0, 0, 1113, 1114, 3, 163, 74, 0, 1114, 1115, 1, 0, 0, 0, 1115, 1116, 6, 118, 24, 0, 1116, 252, 1, 0, 0, 0, 1117, 1118, 7, 12, 0, 0, 1118, 1119, 7, 2, 0, 0, 1119, 254, 1, 0, 0, 0, 1120, 1121, 3, 233, 109, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 120, 25, 0, 1123, 256, 1, 0, 0, 0, 1124, 1125, 3, 57, 21, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 121, 10, 0, 1127, 258, 1, 0, 0, 0, 1128, 1129, 3, 59, 22, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 122, 10, 0, 1131, 260, 1, 0, 0, 0, 1132, 1133, 3, 61, 23, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 123, 10, 0, 1135, 262, 1, 0, 0, 0, 1136, 1137, 3, 63, 24, 0, 1137, 1138, 1, 0, 0, 0, 1138, 1139, 6, 124, 15, 0, 1139, 1140, 6, 124, 11, 0, 1140, 264, 1, 0, 0, 0, 1141, 1142, 3, 165, 75, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 125, 13, 0, 1144, 1145, 6, 125, 26, 0, 1145, 266, 1, 0, 0, 0, 1146, 1147, 7, 7, 0, 0, 1147, 1148, 7, 9, 0, 0, 1148, 1149, 1, 0, 0, 0, 1149, 1150, 6, 126, 27, 0, 1150, 268, 1, 0, 0, 0, 1151, 1152, 7, 19, 0, 0, 1152, 1153, 7, 1, 0, 0, 1153, 1154, 7, 5, 0, 0, 1154, 1155, 7, 10, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 127, 27, 0, 1157, 270, 1, 0, 0, 0, 1158, 1159, 8, 34, 0, 0, 1159, 272, 1, 0, 0, 0, 1160, 1162, 3, 271, 128, 0, 1161, 1160, 1, 0, 0, 0, 1162, 1163, 1, 0, 0, 0, 1163, 1161, 1, 0, 0, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 1, 0, 0, 0, 1165, 1166, 3, 337, 161, 0, 1166, 1168, 1, 0, 0, 0, 1167, 1161, 1, 0, 0, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1170, 1, 0, 0, 0, 1169, 1171, 3, 271, 128, 0, 1170, 1169, 1, 0, 0, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1170, 1, 0, 0, 0, 1172, 1173, 1, 0, 0, 0, 1173, 274, 1, 0, 0, 0, 1174, 1175, 3, 273, 129, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 130, 28, 0, 1177, 276, 1, 0, 0, 0, 1178, 1179, 3, 57, 21, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 131, 10, 0, 1181, 278, 1, 0, 0, 0, 1182, 1183, 3, 59, 22, 0, 1183, 1184, 1, 0, 0, 0, 1184, 1185, 6, 132, 10, 0, 1185, 280, 1, 0, 0, 0, 1186, 1187, 3, 61, 23, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1189, 6, 133, 10, 0, 1189, 282, 1, 0, 0, 0, 1190, 1191, 3, 63, 24, 0, 1191, 1192, 1, 0, 0, 0, 1192, 1193, 6, 134, 15, 0, 1193, 1194, 6, 134, 11, 0, 1194, 1195, 6, 134, 11, 0, 1195, 284, 1, 0, 0, 0, 1196, 1197, 3, 97, 41, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 135, 19, 0, 1199, 286, 1, 0, 0, 0, 1200, 1201, 3, 101, 43, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 136, 18, 0, 1203, 288, 1, 0, 0, 0, 1204, 1205, 3, 105, 45, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 137, 22, 0, 1207, 290, 1, 0, 0, 0, 1208, 1209, 3, 269, 127, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 138, 29, 0, 1211, 292, 1, 0, 0, 0, 1212, 1213, 3, 233, 109, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 139, 25, 0, 1215, 294, 1, 0, 0, 0, 1216, 1217, 3, 173, 79, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 140, 30, 0, 1219, 296, 1, 0, 0, 0, 1220, 1221, 3, 129, 57, 0, 1221, 1222, 1, 0, 0, 0, 1222, 1223, 6, 141, 23, 0, 1223, 298, 1, 0, 0, 0, 1224, 1225, 3, 163, 74, 0, 1225, 1226, 1, 0, 0, 0, 1226, 1227, 6, 142, 24, 0, 1227, 300, 1, 0, 0, 0, 1228, 1229, 3, 57, 21, 0, 1229, 1230, 1, 0, 0, 0, 1230, 1231, 6, 143, 10, 0, 1231, 302, 1, 0, 0, 0, 1232, 1233, 3, 59, 22, 0, 1233, 1234, 1, 0, 0, 0, 1234, 1235, 6, 144, 10, 0, 1235, 304, 1, 0, 0, 0, 1236, 1237, 3, 61, 23, 0, 1237, 1238, 1, 0, 0, 0, 1238, 1239, 6, 145, 10, 0, 1239, 306, 1, 0, 0, 0, 1240, 1241, 3, 63, 24, 0, 1241, 1242, 1, 0, 0, 0, 1242, 1243, 6, 146, 15, 0, 1243, 1244, 6, 146, 11, 0, 1244, 308, 1, 0, 0, 0, 1245, 1246, 3, 105, 45, 0, 1246, 1247, 1, 0, 0, 0, 1247, 1248, 6, 147, 22, 0, 1248, 310, 1, 0, 0, 0, 1249, 1250, 3, 129, 57, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 6, 148, 23, 0, 1252, 312, 1, 0, 0, 0, 1253, 1254, 3, 163, 74, 0, 1254, 1255, 1, 0, 0, 0, 1255, 1256, 6, 149, 24, 0, 1256, 314, 1, 0, 0, 0, 1257, 1258, 3, 173, 79, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 150, 30, 0, 1260, 316, 1, 0, 0, 0, 1261, 1262, 3, 169, 77, 0, 1262, 1263, 1, 0, 0, 0, 1263, 1264, 6, 151, 31, 0, 1264, 318, 1, 0, 0, 0, 1265, 1266, 3, 57, 21, 0, 1266, 1267, 1, 0, 0, 0, 1267, 1268, 6, 152, 10, 0, 1268, 320, 1, 0, 0, 0, 1269, 1270, 3, 59, 22, 0, 1270, 1271, 1, 0, 0, 0, 1271, 1272, 6, 153, 10, 0, 1272, 322, 1, 0, 0, 0, 1273, 1274, 3, 61, 23, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 154, 10, 0, 1276, 324, 1, 0, 0, 0, 1277, 1278, 3, 63, 24, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 155, 15, 0, 1280, 1281, 6, 155, 11, 0, 1281, 326, 1, 0, 0, 0, 1282, 1283, 7, 1, 0, 0, 1283, 1284, 7, 9, 0, 0, 1284, 1285, 7, 15, 0, 0, 1285, 1286, 7, 7, 0, 0, 1286, 328, 1, 0, 0, 0, 1287, 1288, 3, 57, 21, 0, 1288, 1289, 1, 0, 0, 0, 1289, 1290, 6, 157, 10, 0, 1290, 330, 1, 0, 0, 0, 1291, 1292, 3, 59, 22, 0, 1292, 1293, 1, 0, 0, 0, 1293, 1294, 6, 158, 10, 0, 1294, 332, 1, 0, 0, 0, 1295, 1296, 3, 61, 23, 0, 1296, 1297, 1, 0, 0, 0, 1297, 1298, 6, 159, 10, 0, 1298, 334, 1, 0, 0, 0, 1299, 1300, 3, 167, 76, 0, 1300, 1301, 1, 0, 0, 0, 1301, 1302, 6, 160, 16, 0, 1302, 1303, 6, 160, 11, 0, 1303, 336, 1, 0, 0, 0, 1304, 1305, 5, 58, 0, 0, 1305, 338, 1, 0, 0, 0, 1306, 1312, 3, 75, 30, 0, 1307, 1312, 3, 65, 25, 0, 1308, 1312, 3, 105, 45, 0, 1309, 1312, 3, 67, 26, 0, 1310, 1312, 3, 81, 33, 0, 1311, 1306, 1, 0, 0, 0, 1311, 1307, 1, 0, 0, 0, 1311, 1308, 1, 0, 0, 0, 1311, 1309, 1, 0, 0, 0, 1311, 1310, 1, 0, 0, 0, 1312, 1313, 1, 0, 0, 0, 1313, 1311, 1, 0, 0, 0, 1313, 1314, 1, 0, 0, 0, 1314, 340, 1, 0, 0, 0, 1315, 1316, 3, 57, 21, 0, 1316, 1317, 1, 0, 0, 0, 1317, 1318, 6, 163, 10, 0, 1318, 342, 1, 0, 0, 0, 1319, 1320, 3, 59, 22, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 164, 10, 0, 1322, 344, 1, 0, 0, 0, 1323, 1324, 3, 61, 23, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1326, 6, 165, 10, 0, 1326, 346, 1, 0, 0, 0, 1327, 1328, 3, 63, 24, 0, 1328, 1329, 1, 0, 0, 0, 1329, 1330, 6, 166, 15, 0, 1330, 1331, 6, 166, 11, 0, 1331, 348, 1, 0, 0, 0, 1332, 1333, 3, 337, 161, 0, 1333, 1334, 1, 0, 0, 0, 1334, 1335, 6, 167, 17, 0, 1335, 350, 1, 0, 0, 0, 1336, 1337, 3, 101, 43, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 168, 18, 0, 1339, 352, 1, 0, 0, 0, 1340, 1341, 3, 105, 45, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 169, 22, 0, 1343, 354, 1, 0, 0, 0, 1344, 1345, 3, 267, 126, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 170, 32, 0, 1347, 1348, 6, 170, 33, 0, 1348, 356, 1, 0, 0, 0, 1349, 1350, 3, 207, 96, 0, 1350, 1351, 1, 0, 0, 0, 1351, 1352, 6, 171, 20, 0, 1352, 358, 1, 0, 0, 0, 1353, 1354, 3, 85, 35, 0, 1354, 1355, 1, 0, 0, 0, 1355, 1356, 6, 172, 21, 0, 1356, 360, 1, 0, 0, 0, 1357, 1358, 3, 57, 21, 0, 1358, 1359, 1, 0, 0, 0, 1359, 1360, 6, 173, 10, 0, 1360, 362, 1, 0, 0, 0, 1361, 1362, 3, 59, 22, 0, 1362, 1363, 1, 0, 0, 0, 1363, 1364, 6, 174, 10, 0, 1364, 364, 1, 0, 0, 0, 1365, 1366, 3, 61, 23, 0, 1366, 1367, 1, 0, 0, 0, 1367, 1368, 6, 175, 10, 0, 1368, 366, 1, 0, 0, 0, 1369, 1370, 3, 63, 24, 0, 1370, 1371, 1, 0, 0, 0, 1371, 1372, 6, 176, 15, 0, 1372, 1373, 6, 176, 11, 0, 1373, 1374, 6, 176, 11, 0, 1374, 368, 1, 0, 0, 0, 1375, 1376, 3, 101, 43, 0, 1376, 1377, 1, 0, 0, 0, 1377, 1378, 6, 177, 18, 0, 1378, 370, 1, 0, 0, 0, 1379, 1380, 3, 105, 45, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 178, 22, 0, 1382, 372, 1, 0, 0, 0, 1383, 1384, 3, 233, 109, 0, 1384, 1385, 1, 0, 0, 0, 1385, 1386, 6, 179, 25, 0, 1386, 374, 1, 0, 0, 0, 1387, 1388, 3, 57, 21, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 180, 10, 0, 1390, 376, 1, 0, 0, 0, 1391, 1392, 3, 59, 22, 0, 1392, 1393, 1, 0, 0, 0, 1393, 1394, 6, 181, 10, 0, 1394, 378, 1, 0, 0, 0, 1395, 1396, 3, 61, 23, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 182, 10, 0, 1398, 380, 1, 0, 0, 0, 1399, 1400, 3, 63, 24, 0, 1400, 1401, 1, 0, 0, 0, 1401, 1402, 6, 183, 15, 0, 1402, 1403, 6, 183, 11, 0, 1403, 382, 1, 0, 0, 0, 1404, 1405, 3, 207, 96, 0, 1405, 1406, 1, 0, 0, 0, 1406, 1407, 6, 184, 20, 0, 1407, 1408, 6, 184, 11, 0, 1408, 1409, 6, 184, 34, 0, 1409, 384, 1, 0, 0, 0, 1410, 1411, 3, 85, 35, 0, 1411, 1412, 1, 0, 0, 0, 1412, 1413, 6, 185, 21, 0, 1413, 1414, 6, 185, 11, 0, 1414, 1415, 6, 185, 34, 0, 1415, 386, 1, 0, 0, 0, 1416, 1417, 3, 57, 21, 0, 1417, 1418, 1, 0, 0, 0, 1418, 1419, 6, 186, 10, 0, 1419, 388, 1, 0, 0, 0, 1420, 1421, 3, 59, 22, 0, 1421, 1422, 1, 0, 0, 0, 1422, 1423, 6, 187, 10, 0, 1423, 390, 1, 0, 0, 0, 1424, 1425, 3, 61, 23, 0, 1425, 1426, 1, 0, 0, 0, 1426, 1427, 6, 188, 10, 0, 1427, 392, 1, 0, 0, 0, 1428, 1429, 3, 337, 161, 0, 1429, 1430, 1, 0, 0, 0, 1430, 1431, 6, 189, 17, 0, 1431, 1432, 6, 189, 11, 0, 1432, 1433, 6, 189, 9, 0, 1433, 394, 1, 0, 0, 0, 1434, 1435, 3, 101, 43, 0, 1435, 1436, 1, 0, 0, 0, 1436, 1437, 6, 190, 18, 0, 1437, 1438, 6, 190, 11, 0, 1438, 1439, 6, 190, 9, 0, 1439, 396, 1, 0, 0, 0, 1440, 1441, 3, 57, 21, 0, 1441, 1442, 1, 0, 0, 0, 1442, 1443, 6, 191, 10, 0, 1443, 398, 1, 0, 0, 0, 1444, 1445, 3, 59, 22, 0, 1445, 1446, 1, 0, 0, 0, 1446, 1447, 6, 192, 10, 0, 1447, 400, 1, 0, 0, 0, 1448, 1449, 3, 61, 23, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 193, 10, 0, 1451, 402, 1, 0, 0, 0, 1452, 1453, 3, 173, 79, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 194, 11, 0, 1455, 1456, 6, 194, 0, 0, 1456, 1457, 6, 194, 30, 0, 1457, 404, 1, 0, 0, 0, 1458, 1459, 3, 169, 77, 0, 1459, 1460, 1, 0, 0, 0, 1460, 1461, 6, 195, 11, 0, 1461, 1462, 6, 195, 0, 0, 1462, 1463, 6, 195, 31, 0, 1463, 406, 1, 0, 0, 0, 1464, 1465, 3, 91, 38, 0, 1465, 1466, 1, 0, 0, 0, 1466, 1467, 6, 196, 11, 0, 1467, 1468, 6, 196, 0, 0, 1468, 1469, 6, 196, 35, 0, 1469, 408, 1, 0, 0, 0, 1470, 1471, 3, 63, 24, 0, 1471, 1472, 1, 0, 0, 0, 1472, 1473, 6, 197, 15, 0, 1473, 1474, 6, 197, 11, 0, 1474, 410, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 588, 598, 602, 605, 614, 616, 627, 646, 651, 660, 667, 672, 674, 685, 693, 696, 698, 703, 708, 714, 721, 726, 732, 735, 743, 747, 874, 879, 886, 888, 904, 909, 914, 916, 922, 999, 1004, 1051, 1055, 1060, 1065, 1070, 1072, 1076, 1078, 1163, 1167, 1172, 1311, 1313, 36, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 19, 0, 7, 65, 0, 5, 0, 0, 7, 25, 0, 7, 66, 0, 7, 104, 0, 7, 34, 0, 7, 32, 0, 7, 76, 0, 7, 26, 0, 7, 36, 0, 7, 48, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 29, 0]
\ No newline at end of file
+[4, 0, 120, 1479, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 4, 19, 578, 8, 19, 11, 19, 12, 19, 579, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 588, 8, 20, 10, 20, 12, 20, 591, 9, 20, 1, 20, 3, 20, 594, 8, 20, 1, 20, 3, 20, 597, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 606, 8, 21, 10, 21, 12, 21, 609, 9, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 4, 22, 617, 8, 22, 11, 22, 12, 22, 618, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 3, 28, 638, 8, 28, 1, 28, 4, 28, 641, 8, 28, 11, 28, 12, 28, 642, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 652, 8, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 659, 8, 33, 1, 34, 1, 34, 1, 34, 5, 34, 664, 8, 34, 10, 34, 12, 34, 667, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 675, 8, 34, 10, 34, 12, 34, 678, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 685, 8, 34, 1, 34, 3, 34, 688, 8, 34, 3, 34, 690, 8, 34, 1, 35, 4, 35, 693, 8, 35, 11, 35, 12, 35, 694, 1, 36, 4, 36, 698, 8, 36, 11, 36, 12, 36, 699, 1, 36, 1, 36, 5, 36, 704, 8, 36, 10, 36, 12, 36, 707, 9, 36, 1, 36, 1, 36, 4, 36, 711, 8, 36, 11, 36, 12, 36, 712, 1, 36, 4, 36, 716, 8, 36, 11, 36, 12, 36, 717, 1, 36, 1, 36, 5, 36, 722, 8, 36, 10, 36, 12, 36, 725, 9, 36, 3, 36, 727, 8, 36, 1, 36, 1, 36, 1, 36, 1, 36, 4, 36, 733, 8, 36, 11, 36, 12, 36, 734, 1, 36, 1, 36, 3, 36, 739, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 871, 8, 74, 1, 74, 5, 74, 874, 8, 74, 10, 74, 12, 74, 877, 9, 74, 1, 74, 1, 74, 4, 74, 881, 8, 74, 11, 74, 12, 74, 882, 3, 74, 885, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 899, 8, 77, 10, 77, 12, 77, 902, 9, 77, 1, 77, 1, 77, 3, 77, 906, 8, 77, 1, 77, 4, 77, 909, 8, 77, 11, 77, 12, 77, 910, 3, 77, 913, 8, 77, 1, 78, 1, 78, 4, 78, 917, 8, 78, 11, 78, 12, 78, 918, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 996, 8, 95, 1, 96, 4, 96, 999, 8, 96, 11, 96, 12, 96, 1000, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 3, 107, 1050, 8, 107, 1, 108, 1, 108, 3, 108, 1054, 8, 108, 1, 108, 5, 108, 1057, 8, 108, 10, 108, 12, 108, 1060, 9, 108, 1, 108, 1, 108, 3, 108, 1064, 8, 108, 1, 108, 4, 108, 1067, 8, 108, 11, 108, 12, 108, 1068, 3, 108, 1071, 8, 108, 1, 109, 1, 109, 4, 109, 1075, 8, 109, 11, 109, 12, 109, 1076, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 129, 4, 129, 1162, 8, 129, 11, 129, 12, 129, 1163, 1, 129, 1, 129, 3, 129, 1168, 8, 129, 1, 129, 4, 129, 1171, 8, 129, 11, 129, 12, 129, 1172, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 4, 162, 1316, 8, 162, 11, 162, 12, 162, 1317, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 2, 607, 676, 0, 198, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 0, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 25, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 0, 233, 80, 235, 81, 237, 82, 239, 83, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 84, 255, 0, 257, 85, 259, 86, 261, 87, 263, 0, 265, 0, 267, 88, 269, 89, 271, 0, 273, 90, 275, 0, 277, 91, 279, 92, 281, 93, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 0, 301, 94, 303, 95, 305, 96, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 97, 321, 98, 323, 99, 325, 0, 327, 100, 329, 101, 331, 102, 333, 103, 335, 0, 337, 104, 339, 105, 341, 106, 343, 107, 345, 108, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 0, 361, 109, 363, 110, 365, 111, 367, 0, 369, 0, 371, 0, 373, 0, 375, 112, 377, 113, 379, 114, 381, 0, 383, 0, 385, 0, 387, 115, 389, 116, 391, 117, 393, 0, 395, 0, 397, 118, 399, 119, 401, 120, 403, 0, 405, 0, 407, 0, 409, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1507, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 1, 83, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 227, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 9, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 10, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 12, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 13, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 14, 409, 1, 0, 0, 0, 15, 411, 1, 0, 0, 0, 17, 421, 1, 0, 0, 0, 19, 428, 1, 0, 0, 0, 21, 437, 1, 0, 0, 0, 23, 444, 1, 0, 0, 0, 25, 454, 1, 0, 0, 0, 27, 461, 1, 0, 0, 0, 29, 468, 1, 0, 0, 0, 31, 475, 1, 0, 0, 0, 33, 483, 1, 0, 0, 0, 35, 495, 1, 0, 0, 0, 37, 504, 1, 0, 0, 0, 39, 510, 1, 0, 0, 0, 41, 517, 1, 0, 0, 0, 43, 524, 1, 0, 0, 0, 45, 532, 1, 0, 0, 0, 47, 540, 1, 0, 0, 0, 49, 555, 1, 0, 0, 0, 51, 565, 1, 0, 0, 0, 53, 577, 1, 0, 0, 0, 55, 583, 1, 0, 0, 0, 57, 600, 1, 0, 0, 0, 59, 616, 1, 0, 0, 0, 61, 622, 1, 0, 0, 0, 63, 626, 1, 0, 0, 0, 65, 628, 1, 0, 0, 0, 67, 630, 1, 0, 0, 0, 69, 633, 1, 0, 0, 0, 71, 635, 1, 0, 0, 0, 73, 644, 1, 0, 0, 0, 75, 646, 1, 0, 0, 0, 77, 651, 1, 0, 0, 0, 79, 653, 1, 0, 0, 0, 81, 658, 1, 0, 0, 0, 83, 689, 1, 0, 0, 0, 85, 692, 1, 0, 0, 0, 87, 738, 1, 0, 0, 0, 89, 740, 1, 0, 0, 0, 91, 743, 1, 0, 0, 0, 93, 747, 1, 0, 0, 0, 95, 751, 1, 0, 0, 0, 97, 753, 1, 0, 0, 0, 99, 756, 1, 0, 0, 0, 101, 758, 1, 0, 0, 0, 103, 763, 1, 0, 0, 0, 105, 765, 1, 0, 0, 0, 107, 771, 1, 0, 0, 0, 109, 777, 1, 0, 0, 0, 111, 780, 1, 0, 0, 0, 113, 783, 1, 0, 0, 0, 115, 788, 1, 0, 0, 0, 117, 793, 1, 0, 0, 0, 119, 795, 1, 0, 0, 0, 121, 799, 1, 0, 0, 0, 123, 804, 1, 0, 0, 0, 125, 810, 1, 0, 0, 0, 127, 813, 1, 0, 0, 0, 129, 815, 1, 0, 0, 0, 131, 821, 1, 0, 0, 0, 133, 823, 1, 0, 0, 0, 135, 828, 1, 0, 0, 0, 137, 831, 1, 0, 0, 0, 139, 834, 1, 0, 0, 0, 141, 837, 1, 0, 0, 0, 143, 839, 1, 0, 0, 0, 145, 842, 1, 0, 0, 0, 147, 844, 1, 0, 0, 0, 149, 847, 1, 0, 0, 0, 151, 849, 1, 0, 0, 0, 153, 851, 1, 0, 0, 0, 155, 853, 1, 0, 0, 0, 157, 855, 1, 0, 0, 0, 159, 857, 1, 0, 0, 0, 161, 863, 1, 0, 0, 0, 163, 884, 1, 0, 0, 0, 165, 886, 1, 0, 0, 0, 167, 891, 1, 0, 0, 0, 169, 912, 1, 0, 0, 0, 171, 914, 1, 0, 0, 0, 173, 922, 1, 0, 0, 0, 175, 924, 1, 0, 0, 0, 177, 928, 1, 0, 0, 0, 179, 932, 1, 0, 0, 0, 181, 936, 1, 0, 0, 0, 183, 941, 1, 0, 0, 0, 185, 946, 1, 0, 0, 0, 187, 950, 1, 0, 0, 0, 189, 954, 1, 0, 0, 0, 191, 958, 1, 0, 0, 0, 193, 963, 1, 0, 0, 0, 195, 967, 1, 0, 0, 0, 197, 971, 1, 0, 0, 0, 199, 975, 1, 0, 0, 0, 201, 979, 1, 0, 0, 0, 203, 983, 1, 0, 0, 0, 205, 995, 1, 0, 0, 0, 207, 998, 1, 0, 0, 0, 209, 1002, 1, 0, 0, 0, 211, 1006, 1, 0, 0, 0, 213, 1010, 1, 0, 0, 0, 215, 1014, 1, 0, 0, 0, 217, 1018, 1, 0, 0, 0, 219, 1022, 1, 0, 0, 0, 221, 1027, 1, 0, 0, 0, 223, 1031, 1, 0, 0, 0, 225, 1035, 1, 0, 0, 0, 227, 1040, 1, 0, 0, 0, 229, 1049, 1, 0, 0, 0, 231, 1070, 1, 0, 0, 0, 233, 1074, 1, 0, 0, 0, 235, 1078, 1, 0, 0, 0, 237, 1082, 1, 0, 0, 0, 239, 1086, 1, 0, 0, 0, 241, 1090, 1, 0, 0, 0, 243, 1095, 1, 0, 0, 0, 245, 1099, 1, 0, 0, 0, 247, 1103, 1, 0, 0, 0, 249, 1107, 1, 0, 0, 0, 251, 1112, 1, 0, 0, 0, 253, 1117, 1, 0, 0, 0, 255, 1120, 1, 0, 0, 0, 257, 1124, 1, 0, 0, 0, 259, 1128, 1, 0, 0, 0, 261, 1132, 1, 0, 0, 0, 263, 1136, 1, 0, 0, 0, 265, 1141, 1, 0, 0, 0, 267, 1146, 1, 0, 0, 0, 269, 1151, 1, 0, 0, 0, 271, 1158, 1, 0, 0, 0, 273, 1167, 1, 0, 0, 0, 275, 1174, 1, 0, 0, 0, 277, 1178, 1, 0, 0, 0, 279, 1182, 1, 0, 0, 0, 281, 1186, 1, 0, 0, 0, 283, 1190, 1, 0, 0, 0, 285, 1196, 1, 0, 0, 0, 287, 1200, 1, 0, 0, 0, 289, 1204, 1, 0, 0, 0, 291, 1208, 1, 0, 0, 0, 293, 1212, 1, 0, 0, 0, 295, 1216, 1, 0, 0, 0, 297, 1220, 1, 0, 0, 0, 299, 1225, 1, 0, 0, 0, 301, 1230, 1, 0, 0, 0, 303, 1234, 1, 0, 0, 0, 305, 1238, 1, 0, 0, 0, 307, 1242, 1, 0, 0, 0, 309, 1247, 1, 0, 0, 0, 311, 1251, 1, 0, 0, 0, 313, 1256, 1, 0, 0, 0, 315, 1261, 1, 0, 0, 0, 317, 1265, 1, 0, 0, 0, 319, 1269, 1, 0, 0, 0, 321, 1273, 1, 0, 0, 0, 323, 1277, 1, 0, 0, 0, 325, 1281, 1, 0, 0, 0, 327, 1286, 1, 0, 0, 0, 329, 1291, 1, 0, 0, 0, 331, 1295, 1, 0, 0, 0, 333, 1299, 1, 0, 0, 0, 335, 1303, 1, 0, 0, 0, 337, 1308, 1, 0, 0, 0, 339, 1315, 1, 0, 0, 0, 341, 1319, 1, 0, 0, 0, 343, 1323, 1, 0, 0, 0, 345, 1327, 1, 0, 0, 0, 347, 1331, 1, 0, 0, 0, 349, 1336, 1, 0, 0, 0, 351, 1340, 1, 0, 0, 0, 353, 1344, 1, 0, 0, 0, 355, 1348, 1, 0, 0, 0, 357, 1353, 1, 0, 0, 0, 359, 1357, 1, 0, 0, 0, 361, 1361, 1, 0, 0, 0, 363, 1365, 1, 0, 0, 0, 365, 1369, 1, 0, 0, 0, 367, 1373, 1, 0, 0, 0, 369, 1379, 1, 0, 0, 0, 371, 1383, 1, 0, 0, 0, 373, 1387, 1, 0, 0, 0, 375, 1391, 1, 0, 0, 0, 377, 1395, 1, 0, 0, 0, 379, 1399, 1, 0, 0, 0, 381, 1403, 1, 0, 0, 0, 383, 1408, 1, 0, 0, 0, 385, 1414, 1, 0, 0, 0, 387, 1420, 1, 0, 0, 0, 389, 1424, 1, 0, 0, 0, 391, 1428, 1, 0, 0, 0, 393, 1432, 1, 0, 0, 0, 395, 1438, 1, 0, 0, 0, 397, 1444, 1, 0, 0, 0, 399, 1448, 1, 0, 0, 0, 401, 1452, 1, 0, 0, 0, 403, 1456, 1, 0, 0, 0, 405, 1462, 1, 0, 0, 0, 407, 1468, 1, 0, 0, 0, 409, 1474, 1, 0, 0, 0, 411, 412, 7, 0, 0, 0, 412, 413, 7, 1, 0, 0, 413, 414, 7, 2, 0, 0, 414, 415, 7, 2, 0, 0, 415, 416, 7, 3, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 5, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 0, 0, 0, 420, 16, 1, 0, 0, 0, 421, 422, 7, 0, 0, 0, 422, 423, 7, 6, 0, 0, 423, 424, 7, 7, 0, 0, 424, 425, 7, 8, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 1, 1, 0, 427, 18, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 9, 0, 0, 430, 431, 7, 6, 0, 0, 431, 432, 7, 1, 0, 0, 432, 433, 7, 4, 0, 0, 433, 434, 7, 10, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 2, 2, 0, 436, 20, 1, 0, 0, 0, 437, 438, 7, 3, 0, 0, 438, 439, 7, 11, 0, 0, 439, 440, 7, 12, 0, 0, 440, 441, 7, 13, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 3, 0, 0, 443, 22, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 446, 7, 14, 0, 0, 446, 447, 7, 8, 0, 0, 447, 448, 7, 13, 0, 0, 448, 449, 7, 12, 0, 0, 449, 450, 7, 1, 0, 0, 450, 451, 7, 9, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 4, 3, 0, 453, 24, 1, 0, 0, 0, 454, 455, 7, 15, 0, 0, 455, 456, 7, 6, 0, 0, 456, 457, 7, 7, 0, 0, 457, 458, 7, 16, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 5, 4, 0, 460, 26, 1, 0, 0, 0, 461, 462, 7, 17, 0, 0, 462, 463, 7, 6, 0, 0, 463, 464, 7, 7, 0, 0, 464, 465, 7, 18, 0, 0, 465, 466, 1, 0, 0, 0, 466, 467, 6, 6, 0, 0, 467, 28, 1, 0, 0, 0, 468, 469, 7, 18, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 8, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 1, 0, 474, 30, 1, 0, 0, 0, 475, 476, 7, 13, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 16, 0, 0, 478, 479, 7, 1, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 8, 0, 0, 482, 32, 1, 0, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 11, 0, 0, 485, 486, 5, 95, 0, 0, 486, 487, 7, 3, 0, 0, 487, 488, 7, 14, 0, 0, 488, 489, 7, 8, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 9, 0, 0, 491, 492, 7, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 9, 5, 0, 494, 34, 1, 0, 0, 0, 495, 496, 7, 6, 0, 0, 496, 497, 7, 3, 0, 0, 497, 498, 7, 9, 0, 0, 498, 499, 7, 12, 0, 0, 499, 500, 7, 16, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 10, 6, 0, 503, 36, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 506, 7, 7, 0, 0, 506, 507, 7, 19, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 11, 0, 0, 509, 38, 1, 0, 0, 0, 510, 511, 7, 2, 0, 0, 511, 512, 7, 10, 0, 0, 512, 513, 7, 7, 0, 0, 513, 514, 7, 19, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 12, 7, 0, 516, 40, 1, 0, 0, 0, 517, 518, 7, 2, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 13, 0, 0, 523, 42, 1, 0, 0, 0, 524, 525, 7, 2, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 12, 0, 0, 527, 528, 7, 5, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 14, 0, 0, 531, 44, 1, 0, 0, 0, 532, 533, 7, 19, 0, 0, 533, 534, 7, 10, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 7, 6, 0, 0, 536, 537, 7, 3, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 15, 0, 0, 539, 46, 1, 0, 0, 0, 540, 541, 4, 16, 0, 0, 541, 542, 7, 1, 0, 0, 542, 543, 7, 9, 0, 0, 543, 544, 7, 13, 0, 0, 544, 545, 7, 1, 0, 0, 545, 546, 7, 9, 0, 0, 546, 547, 7, 3, 0, 0, 547, 548, 7, 2, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 12, 0, 0, 550, 551, 7, 5, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 16, 0, 0, 554, 48, 1, 0, 0, 0, 555, 556, 4, 17, 1, 0, 556, 557, 7, 13, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 7, 0, 0, 559, 560, 7, 18, 0, 0, 560, 561, 7, 20, 0, 0, 561, 562, 7, 8, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 17, 8, 0, 564, 50, 1, 0, 0, 0, 565, 566, 4, 18, 2, 0, 566, 567, 7, 16, 0, 0, 567, 568, 7, 3, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 6, 0, 0, 570, 571, 7, 1, 0, 0, 571, 572, 7, 4, 0, 0, 572, 573, 7, 2, 0, 0, 573, 574, 1, 0, 0, 0, 574, 575, 6, 18, 9, 0, 575, 52, 1, 0, 0, 0, 576, 578, 8, 21, 0, 0, 577, 576, 1, 0, 0, 0, 578, 579, 1, 0, 0, 0, 579, 577, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 582, 6, 19, 0, 0, 582, 54, 1, 0, 0, 0, 583, 584, 5, 47, 0, 0, 584, 585, 5, 47, 0, 0, 585, 589, 1, 0, 0, 0, 586, 588, 8, 22, 0, 0, 587, 586, 1, 0, 0, 0, 588, 591, 1, 0, 0, 0, 589, 587, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 593, 1, 0, 0, 0, 591, 589, 1, 0, 0, 0, 592, 594, 5, 13, 0, 0, 593, 592, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 596, 1, 0, 0, 0, 595, 597, 5, 10, 0, 0, 596, 595, 1, 0, 0, 0, 596, 597, 1, 0, 0, 0, 597, 598, 1, 0, 0, 0, 598, 599, 6, 20, 10, 0, 599, 56, 1, 0, 0, 0, 600, 601, 5, 47, 0, 0, 601, 602, 5, 42, 0, 0, 602, 607, 1, 0, 0, 0, 603, 606, 3, 57, 21, 0, 604, 606, 9, 0, 0, 0, 605, 603, 1, 0, 0, 0, 605, 604, 1, 0, 0, 0, 606, 609, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 608, 610, 1, 0, 0, 0, 609, 607, 1, 0, 0, 0, 610, 611, 5, 42, 0, 0, 611, 612, 5, 47, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 6, 21, 10, 0, 614, 58, 1, 0, 0, 0, 615, 617, 7, 23, 0, 0, 616, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 620, 1, 0, 0, 0, 620, 621, 6, 22, 10, 0, 621, 60, 1, 0, 0, 0, 622, 623, 5, 124, 0, 0, 623, 624, 1, 0, 0, 0, 624, 625, 6, 23, 11, 0, 625, 62, 1, 0, 0, 0, 626, 627, 7, 24, 0, 0, 627, 64, 1, 0, 0, 0, 628, 629, 7, 25, 0, 0, 629, 66, 1, 0, 0, 0, 630, 631, 5, 92, 0, 0, 631, 632, 7, 26, 0, 0, 632, 68, 1, 0, 0, 0, 633, 634, 8, 27, 0, 0, 634, 70, 1, 0, 0, 0, 635, 637, 7, 3, 0, 0, 636, 638, 7, 28, 0, 0, 637, 636, 1, 0, 0, 0, 637, 638, 1, 0, 0, 0, 638, 640, 1, 0, 0, 0, 639, 641, 3, 63, 24, 0, 640, 639, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 640, 1, 0, 0, 0, 642, 643, 1, 0, 0, 0, 643, 72, 1, 0, 0, 0, 644, 645, 5, 64, 0, 0, 645, 74, 1, 0, 0, 0, 646, 647, 5, 96, 0, 0, 647, 76, 1, 0, 0, 0, 648, 652, 8, 29, 0, 0, 649, 650, 5, 96, 0, 0, 650, 652, 5, 96, 0, 0, 651, 648, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 652, 78, 1, 0, 0, 0, 653, 654, 5, 95, 0, 0, 654, 80, 1, 0, 0, 0, 655, 659, 3, 65, 25, 0, 656, 659, 3, 63, 24, 0, 657, 659, 3, 79, 32, 0, 658, 655, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 658, 657, 1, 0, 0, 0, 659, 82, 1, 0, 0, 0, 660, 665, 5, 34, 0, 0, 661, 664, 3, 67, 26, 0, 662, 664, 3, 69, 27, 0, 663, 661, 1, 0, 0, 0, 663, 662, 1, 0, 0, 0, 664, 667, 1, 0, 0, 0, 665, 663, 1, 0, 0, 0, 665, 666, 1, 0, 0, 0, 666, 668, 1, 0, 0, 0, 667, 665, 1, 0, 0, 0, 668, 690, 5, 34, 0, 0, 669, 670, 5, 34, 0, 0, 670, 671, 5, 34, 0, 0, 671, 672, 5, 34, 0, 0, 672, 676, 1, 0, 0, 0, 673, 675, 8, 22, 0, 0, 674, 673, 1, 0, 0, 0, 675, 678, 1, 0, 0, 0, 676, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 679, 1, 0, 0, 0, 678, 676, 1, 0, 0, 0, 679, 680, 5, 34, 0, 0, 680, 681, 5, 34, 0, 0, 681, 682, 5, 34, 0, 0, 682, 684, 1, 0, 0, 0, 683, 685, 5, 34, 0, 0, 684, 683, 1, 0, 0, 0, 684, 685, 1, 0, 0, 0, 685, 687, 1, 0, 0, 0, 686, 688, 5, 34, 0, 0, 687, 686, 1, 0, 0, 0, 687, 688, 1, 0, 0, 0, 688, 690, 1, 0, 0, 0, 689, 660, 1, 0, 0, 0, 689, 669, 1, 0, 0, 0, 690, 84, 1, 0, 0, 0, 691, 693, 3, 63, 24, 0, 692, 691, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 692, 1, 0, 0, 0, 694, 695, 1, 0, 0, 0, 695, 86, 1, 0, 0, 0, 696, 698, 3, 63, 24, 0, 697, 696, 1, 0, 0, 0, 698, 699, 1, 0, 0, 0, 699, 697, 1, 0, 0, 0, 699, 700, 1, 0, 0, 0, 700, 701, 1, 0, 0, 0, 701, 705, 3, 103, 44, 0, 702, 704, 3, 63, 24, 0, 703, 702, 1, 0, 0, 0, 704, 707, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 705, 706, 1, 0, 0, 0, 706, 739, 1, 0, 0, 0, 707, 705, 1, 0, 0, 0, 708, 710, 3, 103, 44, 0, 709, 711, 3, 63, 24, 0, 710, 709, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 710, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 739, 1, 0, 0, 0, 714, 716, 3, 63, 24, 0, 715, 714, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 715, 1, 0, 0, 0, 717, 718, 1, 0, 0, 0, 718, 726, 1, 0, 0, 0, 719, 723, 3, 103, 44, 0, 720, 722, 3, 63, 24, 0, 721, 720, 1, 0, 0, 0, 722, 725, 1, 0, 0, 0, 723, 721, 1, 0, 0, 0, 723, 724, 1, 0, 0, 0, 724, 727, 1, 0, 0, 0, 725, 723, 1, 0, 0, 0, 726, 719, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0, 728, 729, 3, 71, 28, 0, 729, 739, 1, 0, 0, 0, 730, 732, 3, 103, 44, 0, 731, 733, 3, 63, 24, 0, 732, 731, 1, 0, 0, 0, 733, 734, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 734, 735, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 3, 71, 28, 0, 737, 739, 1, 0, 0, 0, 738, 697, 1, 0, 0, 0, 738, 708, 1, 0, 0, 0, 738, 715, 1, 0, 0, 0, 738, 730, 1, 0, 0, 0, 739, 88, 1, 0, 0, 0, 740, 741, 7, 30, 0, 0, 741, 742, 7, 31, 0, 0, 742, 90, 1, 0, 0, 0, 743, 744, 7, 12, 0, 0, 744, 745, 7, 9, 0, 0, 745, 746, 7, 0, 0, 0, 746, 92, 1, 0, 0, 0, 747, 748, 7, 12, 0, 0, 748, 749, 7, 2, 0, 0, 749, 750, 7, 4, 0, 0, 750, 94, 1, 0, 0, 0, 751, 752, 5, 61, 0, 0, 752, 96, 1, 0, 0, 0, 753, 754, 5, 58, 0, 0, 754, 755, 5, 58, 0, 0, 755, 98, 1, 0, 0, 0, 756, 757, 5, 44, 0, 0, 757, 100, 1, 0, 0, 0, 758, 759, 7, 0, 0, 0, 759, 760, 7, 3, 0, 0, 760, 761, 7, 2, 0, 0, 761, 762, 7, 4, 0, 0, 762, 102, 1, 0, 0, 0, 763, 764, 5, 46, 0, 0, 764, 104, 1, 0, 0, 0, 765, 766, 7, 15, 0, 0, 766, 767, 7, 12, 0, 0, 767, 768, 7, 13, 0, 0, 768, 769, 7, 2, 0, 0, 769, 770, 7, 3, 0, 0, 770, 106, 1, 0, 0, 0, 771, 772, 7, 15, 0, 0, 772, 773, 7, 1, 0, 0, 773, 774, 7, 6, 0, 0, 774, 775, 7, 2, 0, 0, 775, 776, 7, 5, 0, 0, 776, 108, 1, 0, 0, 0, 777, 778, 7, 1, 0, 0, 778, 779, 7, 9, 0, 0, 779, 110, 1, 0, 0, 0, 780, 781, 7, 1, 0, 0, 781, 782, 7, 2, 0, 0, 782, 112, 1, 0, 0, 0, 783, 784, 7, 13, 0, 0, 784, 785, 7, 12, 0, 0, 785, 786, 7, 2, 0, 0, 786, 787, 7, 5, 0, 0, 787, 114, 1, 0, 0, 0, 788, 789, 7, 13, 0, 0, 789, 790, 7, 1, 0, 0, 790, 791, 7, 18, 0, 0, 791, 792, 7, 3, 0, 0, 792, 116, 1, 0, 0, 0, 793, 794, 5, 40, 0, 0, 794, 118, 1, 0, 0, 0, 795, 796, 7, 9, 0, 0, 796, 797, 7, 7, 0, 0, 797, 798, 7, 5, 0, 0, 798, 120, 1, 0, 0, 0, 799, 800, 7, 9, 0, 0, 800, 801, 7, 20, 0, 0, 801, 802, 7, 13, 0, 0, 802, 803, 7, 13, 0, 0, 803, 122, 1, 0, 0, 0, 804, 805, 7, 9, 0, 0, 805, 806, 7, 20, 0, 0, 806, 807, 7, 13, 0, 0, 807, 808, 7, 13, 0, 0, 808, 809, 7, 2, 0, 0, 809, 124, 1, 0, 0, 0, 810, 811, 7, 7, 0, 0, 811, 812, 7, 6, 0, 0, 812, 126, 1, 0, 0, 0, 813, 814, 5, 63, 0, 0, 814, 128, 1, 0, 0, 0, 815, 816, 7, 6, 0, 0, 816, 817, 7, 13, 0, 0, 817, 818, 7, 1, 0, 0, 818, 819, 7, 18, 0, 0, 819, 820, 7, 3, 0, 0, 820, 130, 1, 0, 0, 0, 821, 822, 5, 41, 0, 0, 822, 132, 1, 0, 0, 0, 823, 824, 7, 5, 0, 0, 824, 825, 7, 6, 0, 0, 825, 826, 7, 20, 0, 0, 826, 827, 7, 3, 0, 0, 827, 134, 1, 0, 0, 0, 828, 829, 5, 61, 0, 0, 829, 830, 5, 61, 0, 0, 830, 136, 1, 0, 0, 0, 831, 832, 5, 61, 0, 0, 832, 833, 5, 126, 0, 0, 833, 138, 1, 0, 0, 0, 834, 835, 5, 33, 0, 0, 835, 836, 5, 61, 0, 0, 836, 140, 1, 0, 0, 0, 837, 838, 5, 60, 0, 0, 838, 142, 1, 0, 0, 0, 839, 840, 5, 60, 0, 0, 840, 841, 5, 61, 0, 0, 841, 144, 1, 0, 0, 0, 842, 843, 5, 62, 0, 0, 843, 146, 1, 0, 0, 0, 844, 845, 5, 62, 0, 0, 845, 846, 5, 61, 0, 0, 846, 148, 1, 0, 0, 0, 847, 848, 5, 43, 0, 0, 848, 150, 1, 0, 0, 0, 849, 850, 5, 45, 0, 0, 850, 152, 1, 0, 0, 0, 851, 852, 5, 42, 0, 0, 852, 154, 1, 0, 0, 0, 853, 854, 5, 47, 0, 0, 854, 156, 1, 0, 0, 0, 855, 856, 5, 37, 0, 0, 856, 158, 1, 0, 0, 0, 857, 858, 7, 16, 0, 0, 858, 859, 7, 12, 0, 0, 859, 860, 7, 5, 0, 0, 860, 861, 7, 4, 0, 0, 861, 862, 7, 10, 0, 0, 862, 160, 1, 0, 0, 0, 863, 864, 3, 45, 15, 0, 864, 865, 1, 0, 0, 0, 865, 866, 6, 73, 12, 0, 866, 162, 1, 0, 0, 0, 867, 870, 3, 127, 56, 0, 868, 871, 3, 65, 25, 0, 869, 871, 3, 79, 32, 0, 870, 868, 1, 0, 0, 0, 870, 869, 1, 0, 0, 0, 871, 875, 1, 0, 0, 0, 872, 874, 3, 81, 33, 0, 873, 872, 1, 0, 0, 0, 874, 877, 1, 0, 0, 0, 875, 873, 1, 0, 0, 0, 875, 876, 1, 0, 0, 0, 876, 885, 1, 0, 0, 0, 877, 875, 1, 0, 0, 0, 878, 880, 3, 127, 56, 0, 879, 881, 3, 63, 24, 0, 880, 879, 1, 0, 0, 0, 881, 882, 1, 0, 0, 0, 882, 880, 1, 0, 0, 0, 882, 883, 1, 0, 0, 0, 883, 885, 1, 0, 0, 0, 884, 867, 1, 0, 0, 0, 884, 878, 1, 0, 0, 0, 885, 164, 1, 0, 0, 0, 886, 887, 5, 91, 0, 0, 887, 888, 1, 0, 0, 0, 888, 889, 6, 75, 0, 0, 889, 890, 6, 75, 0, 0, 890, 166, 1, 0, 0, 0, 891, 892, 5, 93, 0, 0, 892, 893, 1, 0, 0, 0, 893, 894, 6, 76, 11, 0, 894, 895, 6, 76, 11, 0, 895, 168, 1, 0, 0, 0, 896, 900, 3, 65, 25, 0, 897, 899, 3, 81, 33, 0, 898, 897, 1, 0, 0, 0, 899, 902, 1, 0, 0, 0, 900, 898, 1, 0, 0, 0, 900, 901, 1, 0, 0, 0, 901, 913, 1, 0, 0, 0, 902, 900, 1, 0, 0, 0, 903, 906, 3, 79, 32, 0, 904, 906, 3, 73, 29, 0, 905, 903, 1, 0, 0, 0, 905, 904, 1, 0, 0, 0, 906, 908, 1, 0, 0, 0, 907, 909, 3, 81, 33, 0, 908, 907, 1, 0, 0, 0, 909, 910, 1, 0, 0, 0, 910, 908, 1, 0, 0, 0, 910, 911, 1, 0, 0, 0, 911, 913, 1, 0, 0, 0, 912, 896, 1, 0, 0, 0, 912, 905, 1, 0, 0, 0, 913, 170, 1, 0, 0, 0, 914, 916, 3, 75, 30, 0, 915, 917, 3, 77, 31, 0, 916, 915, 1, 0, 0, 0, 917, 918, 1, 0, 0, 0, 918, 916, 1, 0, 0, 0, 918, 919, 1, 0, 0, 0, 919, 920, 1, 0, 0, 0, 920, 921, 3, 75, 30, 0, 921, 172, 1, 0, 0, 0, 922, 923, 3, 171, 78, 0, 923, 174, 1, 0, 0, 0, 924, 925, 3, 55, 20, 0, 925, 926, 1, 0, 0, 0, 926, 927, 6, 80, 10, 0, 927, 176, 1, 0, 0, 0, 928, 929, 3, 57, 21, 0, 929, 930, 1, 0, 0, 0, 930, 931, 6, 81, 10, 0, 931, 178, 1, 0, 0, 0, 932, 933, 3, 59, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 82, 10, 0, 935, 180, 1, 0, 0, 0, 936, 937, 3, 165, 75, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 83, 13, 0, 939, 940, 6, 83, 14, 0, 940, 182, 1, 0, 0, 0, 941, 942, 3, 61, 23, 0, 942, 943, 1, 0, 0, 0, 943, 944, 6, 84, 15, 0, 944, 945, 6, 84, 11, 0, 945, 184, 1, 0, 0, 0, 946, 947, 3, 59, 22, 0, 947, 948, 1, 0, 0, 0, 948, 949, 6, 85, 10, 0, 949, 186, 1, 0, 0, 0, 950, 951, 3, 55, 20, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 86, 10, 0, 953, 188, 1, 0, 0, 0, 954, 955, 3, 57, 21, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 87, 10, 0, 957, 190, 1, 0, 0, 0, 958, 959, 3, 61, 23, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 88, 15, 0, 961, 962, 6, 88, 11, 0, 962, 192, 1, 0, 0, 0, 963, 964, 3, 165, 75, 0, 964, 965, 1, 0, 0, 0, 965, 966, 6, 89, 13, 0, 966, 194, 1, 0, 0, 0, 967, 968, 3, 167, 76, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 90, 16, 0, 970, 196, 1, 0, 0, 0, 971, 972, 3, 337, 161, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 91, 17, 0, 974, 198, 1, 0, 0, 0, 975, 976, 3, 99, 42, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 92, 18, 0, 978, 200, 1, 0, 0, 0, 979, 980, 3, 95, 40, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 93, 19, 0, 982, 202, 1, 0, 0, 0, 983, 984, 7, 16, 0, 0, 984, 985, 7, 3, 0, 0, 985, 986, 7, 5, 0, 0, 986, 987, 7, 12, 0, 0, 987, 988, 7, 0, 0, 0, 988, 989, 7, 12, 0, 0, 989, 990, 7, 5, 0, 0, 990, 991, 7, 12, 0, 0, 991, 204, 1, 0, 0, 0, 992, 996, 8, 32, 0, 0, 993, 994, 5, 47, 0, 0, 994, 996, 8, 33, 0, 0, 995, 992, 1, 0, 0, 0, 995, 993, 1, 0, 0, 0, 996, 206, 1, 0, 0, 0, 997, 999, 3, 205, 95, 0, 998, 997, 1, 0, 0, 0, 999, 1000, 1, 0, 0, 0, 1000, 998, 1, 0, 0, 0, 1000, 1001, 1, 0, 0, 0, 1001, 208, 1, 0, 0, 0, 1002, 1003, 3, 207, 96, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1005, 6, 97, 20, 0, 1005, 210, 1, 0, 0, 0, 1006, 1007, 3, 83, 34, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 98, 21, 0, 1009, 212, 1, 0, 0, 0, 1010, 1011, 3, 55, 20, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 99, 10, 0, 1013, 214, 1, 0, 0, 0, 1014, 1015, 3, 57, 21, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 100, 10, 0, 1017, 216, 1, 0, 0, 0, 1018, 1019, 3, 59, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 101, 10, 0, 1021, 218, 1, 0, 0, 0, 1022, 1023, 3, 61, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 102, 15, 0, 1025, 1026, 6, 102, 11, 0, 1026, 220, 1, 0, 0, 0, 1027, 1028, 3, 103, 44, 0, 1028, 1029, 1, 0, 0, 0, 1029, 1030, 6, 103, 22, 0, 1030, 222, 1, 0, 0, 0, 1031, 1032, 3, 99, 42, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1034, 6, 104, 18, 0, 1034, 224, 1, 0, 0, 0, 1035, 1036, 4, 105, 3, 0, 1036, 1037, 3, 127, 56, 0, 1037, 1038, 1, 0, 0, 0, 1038, 1039, 6, 105, 23, 0, 1039, 226, 1, 0, 0, 0, 1040, 1041, 4, 106, 4, 0, 1041, 1042, 3, 163, 74, 0, 1042, 1043, 1, 0, 0, 0, 1043, 1044, 6, 106, 24, 0, 1044, 228, 1, 0, 0, 0, 1045, 1050, 3, 65, 25, 0, 1046, 1050, 3, 63, 24, 0, 1047, 1050, 3, 79, 32, 0, 1048, 1050, 3, 153, 69, 0, 1049, 1045, 1, 0, 0, 0, 1049, 1046, 1, 0, 0, 0, 1049, 1047, 1, 0, 0, 0, 1049, 1048, 1, 0, 0, 0, 1050, 230, 1, 0, 0, 0, 1051, 1054, 3, 65, 25, 0, 1052, 1054, 3, 153, 69, 0, 1053, 1051, 1, 0, 0, 0, 1053, 1052, 1, 0, 0, 0, 1054, 1058, 1, 0, 0, 0, 1055, 1057, 3, 229, 107, 0, 1056, 1055, 1, 0, 0, 0, 1057, 1060, 1, 0, 0, 0, 1058, 1056, 1, 0, 0, 0, 1058, 1059, 1, 0, 0, 0, 1059, 1071, 1, 0, 0, 0, 1060, 1058, 1, 0, 0, 0, 1061, 1064, 3, 79, 32, 0, 1062, 1064, 3, 73, 29, 0, 1063, 1061, 1, 0, 0, 0, 1063, 1062, 1, 0, 0, 0, 1064, 1066, 1, 0, 0, 0, 1065, 1067, 3, 229, 107, 0, 1066, 1065, 1, 0, 0, 0, 1067, 1068, 1, 0, 0, 0, 1068, 1066, 1, 0, 0, 0, 1068, 1069, 1, 0, 0, 0, 1069, 1071, 1, 0, 0, 0, 1070, 1053, 1, 0, 0, 0, 1070, 1063, 1, 0, 0, 0, 1071, 232, 1, 0, 0, 0, 1072, 1075, 3, 231, 108, 0, 1073, 1075, 3, 171, 78, 0, 1074, 1072, 1, 0, 0, 0, 1074, 1073, 1, 0, 0, 0, 1075, 1076, 1, 0, 0, 0, 1076, 1074, 1, 0, 0, 0, 1076, 1077, 1, 0, 0, 0, 1077, 234, 1, 0, 0, 0, 1078, 1079, 3, 55, 20, 0, 1079, 1080, 1, 0, 0, 0, 1080, 1081, 6, 110, 10, 0, 1081, 236, 1, 0, 0, 0, 1082, 1083, 3, 57, 21, 0, 1083, 1084, 1, 0, 0, 0, 1084, 1085, 6, 111, 10, 0, 1085, 238, 1, 0, 0, 0, 1086, 1087, 3, 59, 22, 0, 1087, 1088, 1, 0, 0, 0, 1088, 1089, 6, 112, 10, 0, 1089, 240, 1, 0, 0, 0, 1090, 1091, 3, 61, 23, 0, 1091, 1092, 1, 0, 0, 0, 1092, 1093, 6, 113, 15, 0, 1093, 1094, 6, 113, 11, 0, 1094, 242, 1, 0, 0, 0, 1095, 1096, 3, 95, 40, 0, 1096, 1097, 1, 0, 0, 0, 1097, 1098, 6, 114, 19, 0, 1098, 244, 1, 0, 0, 0, 1099, 1100, 3, 99, 42, 0, 1100, 1101, 1, 0, 0, 0, 1101, 1102, 6, 115, 18, 0, 1102, 246, 1, 0, 0, 0, 1103, 1104, 3, 103, 44, 0, 1104, 1105, 1, 0, 0, 0, 1105, 1106, 6, 116, 22, 0, 1106, 248, 1, 0, 0, 0, 1107, 1108, 4, 117, 5, 0, 1108, 1109, 3, 127, 56, 0, 1109, 1110, 1, 0, 0, 0, 1110, 1111, 6, 117, 23, 0, 1111, 250, 1, 0, 0, 0, 1112, 1113, 4, 118, 6, 0, 1113, 1114, 3, 163, 74, 0, 1114, 1115, 1, 0, 0, 0, 1115, 1116, 6, 118, 24, 0, 1116, 252, 1, 0, 0, 0, 1117, 1118, 7, 12, 0, 0, 1118, 1119, 7, 2, 0, 0, 1119, 254, 1, 0, 0, 0, 1120, 1121, 3, 233, 109, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 120, 25, 0, 1123, 256, 1, 0, 0, 0, 1124, 1125, 3, 55, 20, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 121, 10, 0, 1127, 258, 1, 0, 0, 0, 1128, 1129, 3, 57, 21, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 122, 10, 0, 1131, 260, 1, 0, 0, 0, 1132, 1133, 3, 59, 22, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 123, 10, 0, 1135, 262, 1, 0, 0, 0, 1136, 1137, 3, 61, 23, 0, 1137, 1138, 1, 0, 0, 0, 1138, 1139, 6, 124, 15, 0, 1139, 1140, 6, 124, 11, 0, 1140, 264, 1, 0, 0, 0, 1141, 1142, 3, 165, 75, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 125, 13, 0, 1144, 1145, 6, 125, 26, 0, 1145, 266, 1, 0, 0, 0, 1146, 1147, 7, 7, 0, 0, 1147, 1148, 7, 9, 0, 0, 1148, 1149, 1, 0, 0, 0, 1149, 1150, 6, 126, 27, 0, 1150, 268, 1, 0, 0, 0, 1151, 1152, 7, 19, 0, 0, 1152, 1153, 7, 1, 0, 0, 1153, 1154, 7, 5, 0, 0, 1154, 1155, 7, 10, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 127, 27, 0, 1157, 270, 1, 0, 0, 0, 1158, 1159, 8, 34, 0, 0, 1159, 272, 1, 0, 0, 0, 1160, 1162, 3, 271, 128, 0, 1161, 1160, 1, 0, 0, 0, 1162, 1163, 1, 0, 0, 0, 1163, 1161, 1, 0, 0, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 1, 0, 0, 0, 1165, 1166, 3, 337, 161, 0, 1166, 1168, 1, 0, 0, 0, 1167, 1161, 1, 0, 0, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1170, 1, 0, 0, 0, 1169, 1171, 3, 271, 128, 0, 1170, 1169, 1, 0, 0, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1170, 1, 0, 0, 0, 1172, 1173, 1, 0, 0, 0, 1173, 274, 1, 0, 0, 0, 1174, 1175, 3, 273, 129, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 130, 28, 0, 1177, 276, 1, 0, 0, 0, 1178, 1179, 3, 55, 20, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 131, 10, 0, 1181, 278, 1, 0, 0, 0, 1182, 1183, 3, 57, 21, 0, 1183, 1184, 1, 0, 0, 0, 1184, 1185, 6, 132, 10, 0, 1185, 280, 1, 0, 0, 0, 1186, 1187, 3, 59, 22, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1189, 6, 133, 10, 0, 1189, 282, 1, 0, 0, 0, 1190, 1191, 3, 61, 23, 0, 1191, 1192, 1, 0, 0, 0, 1192, 1193, 6, 134, 15, 0, 1193, 1194, 6, 134, 11, 0, 1194, 1195, 6, 134, 11, 0, 1195, 284, 1, 0, 0, 0, 1196, 1197, 3, 95, 40, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 135, 19, 0, 1199, 286, 1, 0, 0, 0, 1200, 1201, 3, 99, 42, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 136, 18, 0, 1203, 288, 1, 0, 0, 0, 1204, 1205, 3, 103, 44, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 137, 22, 0, 1207, 290, 1, 0, 0, 0, 1208, 1209, 3, 269, 127, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 138, 29, 0, 1211, 292, 1, 0, 0, 0, 1212, 1213, 3, 233, 109, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 139, 25, 0, 1215, 294, 1, 0, 0, 0, 1216, 1217, 3, 173, 79, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 140, 30, 0, 1219, 296, 1, 0, 0, 0, 1220, 1221, 4, 141, 7, 0, 1221, 1222, 3, 127, 56, 0, 1222, 1223, 1, 0, 0, 0, 1223, 1224, 6, 141, 23, 0, 1224, 298, 1, 0, 0, 0, 1225, 1226, 4, 142, 8, 0, 1226, 1227, 3, 163, 74, 0, 1227, 1228, 1, 0, 0, 0, 1228, 1229, 6, 142, 24, 0, 1229, 300, 1, 0, 0, 0, 1230, 1231, 3, 55, 20, 0, 1231, 1232, 1, 0, 0, 0, 1232, 1233, 6, 143, 10, 0, 1233, 302, 1, 0, 0, 0, 1234, 1235, 3, 57, 21, 0, 1235, 1236, 1, 0, 0, 0, 1236, 1237, 6, 144, 10, 0, 1237, 304, 1, 0, 0, 0, 1238, 1239, 3, 59, 22, 0, 1239, 1240, 1, 0, 0, 0, 1240, 1241, 6, 145, 10, 0, 1241, 306, 1, 0, 0, 0, 1242, 1243, 3, 61, 23, 0, 1243, 1244, 1, 0, 0, 0, 1244, 1245, 6, 146, 15, 0, 1245, 1246, 6, 146, 11, 0, 1246, 308, 1, 0, 0, 0, 1247, 1248, 3, 103, 44, 0, 1248, 1249, 1, 0, 0, 0, 1249, 1250, 6, 147, 22, 0, 1250, 310, 1, 0, 0, 0, 1251, 1252, 4, 148, 9, 0, 1252, 1253, 3, 127, 56, 0, 1253, 1254, 1, 0, 0, 0, 1254, 1255, 6, 148, 23, 0, 1255, 312, 1, 0, 0, 0, 1256, 1257, 4, 149, 10, 0, 1257, 1258, 3, 163, 74, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 149, 24, 0, 1260, 314, 1, 0, 0, 0, 1261, 1262, 3, 173, 79, 0, 1262, 1263, 1, 0, 0, 0, 1263, 1264, 6, 150, 30, 0, 1264, 316, 1, 0, 0, 0, 1265, 1266, 3, 169, 77, 0, 1266, 1267, 1, 0, 0, 0, 1267, 1268, 6, 151, 31, 0, 1268, 318, 1, 0, 0, 0, 1269, 1270, 3, 55, 20, 0, 1270, 1271, 1, 0, 0, 0, 1271, 1272, 6, 152, 10, 0, 1272, 320, 1, 0, 0, 0, 1273, 1274, 3, 57, 21, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 153, 10, 0, 1276, 322, 1, 0, 0, 0, 1277, 1278, 3, 59, 22, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 154, 10, 0, 1280, 324, 1, 0, 0, 0, 1281, 1282, 3, 61, 23, 0, 1282, 1283, 1, 0, 0, 0, 1283, 1284, 6, 155, 15, 0, 1284, 1285, 6, 155, 11, 0, 1285, 326, 1, 0, 0, 0, 1286, 1287, 7, 1, 0, 0, 1287, 1288, 7, 9, 0, 0, 1288, 1289, 7, 15, 0, 0, 1289, 1290, 7, 7, 0, 0, 1290, 328, 1, 0, 0, 0, 1291, 1292, 3, 55, 20, 0, 1292, 1293, 1, 0, 0, 0, 1293, 1294, 6, 157, 10, 0, 1294, 330, 1, 0, 0, 0, 1295, 1296, 3, 57, 21, 0, 1296, 1297, 1, 0, 0, 0, 1297, 1298, 6, 158, 10, 0, 1298, 332, 1, 0, 0, 0, 1299, 1300, 3, 59, 22, 0, 1300, 1301, 1, 0, 0, 0, 1301, 1302, 6, 159, 10, 0, 1302, 334, 1, 0, 0, 0, 1303, 1304, 3, 167, 76, 0, 1304, 1305, 1, 0, 0, 0, 1305, 1306, 6, 160, 16, 0, 1306, 1307, 6, 160, 11, 0, 1307, 336, 1, 0, 0, 0, 1308, 1309, 5, 58, 0, 0, 1309, 338, 1, 0, 0, 0, 1310, 1316, 3, 73, 29, 0, 1311, 1316, 3, 63, 24, 0, 1312, 1316, 3, 103, 44, 0, 1313, 1316, 3, 65, 25, 0, 1314, 1316, 3, 79, 32, 0, 1315, 1310, 1, 0, 0, 0, 1315, 1311, 1, 0, 0, 0, 1315, 1312, 1, 0, 0, 0, 1315, 1313, 1, 0, 0, 0, 1315, 1314, 1, 0, 0, 0, 1316, 1317, 1, 0, 0, 0, 1317, 1315, 1, 0, 0, 0, 1317, 1318, 1, 0, 0, 0, 1318, 340, 1, 0, 0, 0, 1319, 1320, 3, 55, 20, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 163, 10, 0, 1322, 342, 1, 0, 0, 0, 1323, 1324, 3, 57, 21, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1326, 6, 164, 10, 0, 1326, 344, 1, 0, 0, 0, 1327, 1328, 3, 59, 22, 0, 1328, 1329, 1, 0, 0, 0, 1329, 1330, 6, 165, 10, 0, 1330, 346, 1, 0, 0, 0, 1331, 1332, 3, 61, 23, 0, 1332, 1333, 1, 0, 0, 0, 1333, 1334, 6, 166, 15, 0, 1334, 1335, 6, 166, 11, 0, 1335, 348, 1, 0, 0, 0, 1336, 1337, 3, 337, 161, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 167, 17, 0, 1339, 350, 1, 0, 0, 0, 1340, 1341, 3, 99, 42, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 168, 18, 0, 1343, 352, 1, 0, 0, 0, 1344, 1345, 3, 103, 44, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 169, 22, 0, 1347, 354, 1, 0, 0, 0, 1348, 1349, 3, 267, 126, 0, 1349, 1350, 1, 0, 0, 0, 1350, 1351, 6, 170, 32, 0, 1351, 1352, 6, 170, 33, 0, 1352, 356, 1, 0, 0, 0, 1353, 1354, 3, 207, 96, 0, 1354, 1355, 1, 0, 0, 0, 1355, 1356, 6, 171, 20, 0, 1356, 358, 1, 0, 0, 0, 1357, 1358, 3, 83, 34, 0, 1358, 1359, 1, 0, 0, 0, 1359, 1360, 6, 172, 21, 0, 1360, 360, 1, 0, 0, 0, 1361, 1362, 3, 55, 20, 0, 1362, 1363, 1, 0, 0, 0, 1363, 1364, 6, 173, 10, 0, 1364, 362, 1, 0, 0, 0, 1365, 1366, 3, 57, 21, 0, 1366, 1367, 1, 0, 0, 0, 1367, 1368, 6, 174, 10, 0, 1368, 364, 1, 0, 0, 0, 1369, 1370, 3, 59, 22, 0, 1370, 1371, 1, 0, 0, 0, 1371, 1372, 6, 175, 10, 0, 1372, 366, 1, 0, 0, 0, 1373, 1374, 3, 61, 23, 0, 1374, 1375, 1, 0, 0, 0, 1375, 1376, 6, 176, 15, 0, 1376, 1377, 6, 176, 11, 0, 1377, 1378, 6, 176, 11, 0, 1378, 368, 1, 0, 0, 0, 1379, 1380, 3, 99, 42, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 177, 18, 0, 1382, 370, 1, 0, 0, 0, 1383, 1384, 3, 103, 44, 0, 1384, 1385, 1, 0, 0, 0, 1385, 1386, 6, 178, 22, 0, 1386, 372, 1, 0, 0, 0, 1387, 1388, 3, 233, 109, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 179, 25, 0, 1390, 374, 1, 0, 0, 0, 1391, 1392, 3, 55, 20, 0, 1392, 1393, 1, 0, 0, 0, 1393, 1394, 6, 180, 10, 0, 1394, 376, 1, 0, 0, 0, 1395, 1396, 3, 57, 21, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 181, 10, 0, 1398, 378, 1, 0, 0, 0, 1399, 1400, 3, 59, 22, 0, 1400, 1401, 1, 0, 0, 0, 1401, 1402, 6, 182, 10, 0, 1402, 380, 1, 0, 0, 0, 1403, 1404, 3, 61, 23, 0, 1404, 1405, 1, 0, 0, 0, 1405, 1406, 6, 183, 15, 0, 1406, 1407, 6, 183, 11, 0, 1407, 382, 1, 0, 0, 0, 1408, 1409, 3, 207, 96, 0, 1409, 1410, 1, 0, 0, 0, 1410, 1411, 6, 184, 20, 0, 1411, 1412, 6, 184, 11, 0, 1412, 1413, 6, 184, 34, 0, 1413, 384, 1, 0, 0, 0, 1414, 1415, 3, 83, 34, 0, 1415, 1416, 1, 0, 0, 0, 1416, 1417, 6, 185, 21, 0, 1417, 1418, 6, 185, 11, 0, 1418, 1419, 6, 185, 34, 0, 1419, 386, 1, 0, 0, 0, 1420, 1421, 3, 55, 20, 0, 1421, 1422, 1, 0, 0, 0, 1422, 1423, 6, 186, 10, 0, 1423, 388, 1, 0, 0, 0, 1424, 1425, 3, 57, 21, 0, 1425, 1426, 1, 0, 0, 0, 1426, 1427, 6, 187, 10, 0, 1427, 390, 1, 0, 0, 0, 1428, 1429, 3, 59, 22, 0, 1429, 1430, 1, 0, 0, 0, 1430, 1431, 6, 188, 10, 0, 1431, 392, 1, 0, 0, 0, 1432, 1433, 3, 337, 161, 0, 1433, 1434, 1, 0, 0, 0, 1434, 1435, 6, 189, 17, 0, 1435, 1436, 6, 189, 11, 0, 1436, 1437, 6, 189, 9, 0, 1437, 394, 1, 0, 0, 0, 1438, 1439, 3, 99, 42, 0, 1439, 1440, 1, 0, 0, 0, 1440, 1441, 6, 190, 18, 0, 1441, 1442, 6, 190, 11, 0, 1442, 1443, 6, 190, 9, 0, 1443, 396, 1, 0, 0, 0, 1444, 1445, 3, 55, 20, 0, 1445, 1446, 1, 0, 0, 0, 1446, 1447, 6, 191, 10, 0, 1447, 398, 1, 0, 0, 0, 1448, 1449, 3, 57, 21, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 192, 10, 0, 1451, 400, 1, 0, 0, 0, 1452, 1453, 3, 59, 22, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 193, 10, 0, 1455, 402, 1, 0, 0, 0, 1456, 1457, 3, 173, 79, 0, 1457, 1458, 1, 0, 0, 0, 1458, 1459, 6, 194, 11, 0, 1459, 1460, 6, 194, 0, 0, 1460, 1461, 6, 194, 30, 0, 1461, 404, 1, 0, 0, 0, 1462, 1463, 3, 169, 77, 0, 1463, 1464, 1, 0, 0, 0, 1464, 1465, 6, 195, 11, 0, 1465, 1466, 6, 195, 0, 0, 1466, 1467, 6, 195, 31, 0, 1467, 406, 1, 0, 0, 0, 1468, 1469, 3, 89, 37, 0, 1469, 1470, 1, 0, 0, 0, 1470, 1471, 6, 196, 11, 0, 1471, 1472, 6, 196, 0, 0, 1472, 1473, 6, 196, 35, 0, 1473, 408, 1, 0, 0, 0, 1474, 1475, 3, 61, 23, 0, 1475, 1476, 1, 0, 0, 0, 1476, 1477, 6, 197, 15, 0, 1477, 1478, 6, 197, 11, 0, 1478, 410, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 579, 589, 593, 596, 605, 607, 618, 637, 642, 651, 658, 663, 665, 676, 684, 687, 689, 694, 699, 705, 712, 717, 723, 726, 734, 738, 870, 875, 882, 884, 900, 905, 910, 912, 918, 995, 1000, 1049, 1053, 1058, 1063, 1068, 1070, 1074, 1076, 1163, 1167, 1172, 1315, 1317, 36, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 65, 0, 5, 0, 0, 7, 24, 0, 7, 66, 0, 7, 104, 0, 7, 33, 0, 7, 31, 0, 7, 76, 0, 7, 25, 0, 7, 35, 0, 7, 47, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 28, 0]
\ No newline at end of file
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
index f67daa29ab059..cef4bc5378aaa 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
@@ -27,13 +27,13 @@ public class EsqlBaseLexer extends LexerConfig {
   public static final int
     DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, 
     LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, 
-    WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, 
-    UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, 
-    QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, 
-    ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, 
-    IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, 
-    PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, 
-    GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, 
+    WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, UNKNOWN_CMD=20, 
+    LINE_COMMENT=21, MULTILINE_COMMENT=22, WS=23, PIPE=24, QUOTED_STRING=25, 
+    INTEGER_LITERAL=26, DECIMAL_LITERAL=27, BY=28, AND=29, ASC=30, ASSIGN=31, 
+    CAST_OP=32, COMMA=33, DESC=34, DOT=35, FALSE=36, FIRST=37, IN=38, IS=39, 
+    LAST=40, LIKE=41, LP=42, NOT=43, NULL=44, NULLS=45, OR=46, PARAM=47, RLIKE=48, 
+    RP=49, TRUE=50, EQ=51, CIEQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, 
+    PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, MATCH=63, NAMED_OR_POSITIONAL_PARAM=64, 
     OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, 
     EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, 
     EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, 
@@ -68,15 +68,15 @@ private static String[] makeRuleNames() {
     return new String[] {
       "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", 
       "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", 
-      "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", "UNKNOWN_CMD", 
-      "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", 
-      "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", 
-      "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", 
-      "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
-      "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", 
-      "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", 
-      "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", 
-      "SLASH", "PERCENT", "DEV_MATCH_OP", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", 
+      "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", "LINE_COMMENT", 
+      "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", 
+      "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", 
+      "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", 
+      "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", 
+      "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", 
+      "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", 
+      "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", 
+      "PERCENT", "MATCH", "NESTED_WHERE", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", 
       "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", 
       "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_OPENING_BRACKET", 
       "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", 
@@ -119,15 +119,15 @@ private static String[] makeLiteralNames() {
       null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", 
       "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", 
       "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, 
-      null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", 
-      "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", 
-      "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", 
-      "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", 
-      "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, 
-      null, null, null, null, null, "'metadata'", null, null, null, null, null, 
-      null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, 
-      null, null, null, null, null, null, null, null, "'info'", null, null, 
-      null, "':'"
+      "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", 
+      "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", 
+      "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", 
+      "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", 
+      "'-'", "'*'", "'/'", "'%'", "'match'", null, null, "']'", null, null, 
+      null, null, null, null, null, null, "'metadata'", null, null, null, null, 
+      null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, 
+      null, null, null, null, null, null, null, null, null, "'info'", null, 
+      null, null, "':'"
     };
   }
   private static final String[] _LITERAL_NAMES = makeLiteralNames();
@@ -135,13 +135,13 @@ private static String[] makeSymbolicNames() {
     return new String[] {
       null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", 
       "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", 
-      "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", 
-      "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", 
-      "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
-      "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", 
-      "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", 
-      "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", 
-      "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
+      "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", 
+      "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", 
+      "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", 
+      "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", 
+      "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", 
+      "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", 
+      "PERCENT", "MATCH", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
       "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", 
       "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", 
       "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", 
@@ -226,11 +226,23 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     case 17:
       return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex);
     case 18:
-      return DEV_MATCH_sempred((RuleContext)_localctx, predIndex);
-    case 19:
       return DEV_METRICS_sempred((RuleContext)_localctx, predIndex);
-    case 73:
-      return DEV_MATCH_OP_sempred((RuleContext)_localctx, predIndex);
+    case 105:
+      return PROJECT_PARAM_sempred((RuleContext)_localctx, predIndex);
+    case 106:
+      return PROJECT_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex);
+    case 117:
+      return RENAME_PARAM_sempred((RuleContext)_localctx, predIndex);
+    case 118:
+      return RENAME_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex);
+    case 141:
+      return ENRICH_FIELD_PARAM_sempred((RuleContext)_localctx, predIndex);
+    case 142:
+      return ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex);
+    case 148:
+      return MVEXPAND_PARAM_sempred((RuleContext)_localctx, predIndex);
+    case 149:
+      return MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
   }
@@ -248,30 +260,72 @@ private boolean DEV_LOOKUP_sempred(RuleContext _localctx, int predIndex) {
     }
     return true;
   }
-  private boolean DEV_MATCH_sempred(RuleContext _localctx, int predIndex) {
+  private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 2:
       return this.isDevVersion();
     }
     return true;
   }
-  private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) {
+  private boolean PROJECT_PARAM_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 3:
       return this.isDevVersion();
     }
     return true;
   }
-  private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) {
+  private boolean PROJECT_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 4:
       return this.isDevVersion();
     }
     return true;
   }
+  private boolean RENAME_PARAM_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 5:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean RENAME_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 6:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean ENRICH_FIELD_PARAM_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 7:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 8:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean MVEXPAND_PARAM_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 9:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 10:
+      return this.isDevVersion();
+    }
+    return true;
+  }
 
   public static final String _serializedATN =
-    "\u0004\u0000x\u05c3\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
+    "\u0004\u0000x\u05c7\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
@@ -352,67 +406,67 @@ private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) {
     "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011"+
     "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+
     "\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+
-    "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+
-    "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+
-    "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0004\u0014\u024b\b\u0014"+
-    "\u000b\u0014\f\u0014\u024c\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+
-    "\u0001\u0015\u0001\u0015\u0005\u0015\u0255\b\u0015\n\u0015\f\u0015\u0258"+
-    "\t\u0015\u0001\u0015\u0003\u0015\u025b\b\u0015\u0001\u0015\u0003\u0015"+
-    "\u025e\b\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+
-    "\u0001\u0016\u0001\u0016\u0005\u0016\u0267\b\u0016\n\u0016\f\u0016\u026a"+
-    "\t\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001"+
-    "\u0017\u0004\u0017\u0272\b\u0017\u000b\u0017\f\u0017\u0273\u0001\u0017"+
-    "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"+
-    "\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+
-    "\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0003\u001d\u0287\b\u001d"+
-    "\u0001\u001d\u0004\u001d\u028a\b\u001d\u000b\u001d\f\u001d\u028b\u0001"+
-    "\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001 \u0003"+
-    " \u0295\b \u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0003\"\u029c\b\"\u0001"+
-    "#\u0001#\u0001#\u0005#\u02a1\b#\n#\f#\u02a4\t#\u0001#\u0001#\u0001#\u0001"+
-    "#\u0001#\u0001#\u0005#\u02ac\b#\n#\f#\u02af\t#\u0001#\u0001#\u0001#\u0001"+
-    "#\u0001#\u0003#\u02b6\b#\u0001#\u0003#\u02b9\b#\u0003#\u02bb\b#\u0001"+
-    "$\u0004$\u02be\b$\u000b$\f$\u02bf\u0001%\u0004%\u02c3\b%\u000b%\f%\u02c4"+
-    "\u0001%\u0001%\u0005%\u02c9\b%\n%\f%\u02cc\t%\u0001%\u0001%\u0004%\u02d0"+
-    "\b%\u000b%\f%\u02d1\u0001%\u0004%\u02d5\b%\u000b%\f%\u02d6\u0001%\u0001"+
-    "%\u0005%\u02db\b%\n%\f%\u02de\t%\u0003%\u02e0\b%\u0001%\u0001%\u0001%"+
-    "\u0001%\u0004%\u02e6\b%\u000b%\f%\u02e7\u0001%\u0001%\u0003%\u02ec\b%"+
-    "\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001"+
-    "(\u0001(\u0001)\u0001)\u0001*\u0001*\u0001*\u0001+\u0001+\u0001,\u0001"+
-    ",\u0001,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001"+
-    ".\u0001.\u0001/\u0001/\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u0001"+
-    "0\u00011\u00011\u00011\u00012\u00012\u00012\u00012\u00012\u00013\u0001"+
-    "3\u00013\u00013\u00013\u00014\u00014\u00015\u00015\u00015\u00015\u0001"+
-    "6\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u00017\u0001"+
-    "7\u00018\u00018\u00018\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001"+
-    ":\u0001:\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001"+
-    "=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001"+
-    "A\u0001A\u0001A\u0001B\u0001B\u0001C\u0001C\u0001C\u0001D\u0001D\u0001"+
-    "E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001"+
-    "I\u0001I\u0001I\u0001J\u0001J\u0001J\u0003J\u036b\bJ\u0001J\u0005J\u036e"+
-    "\bJ\nJ\fJ\u0371\tJ\u0001J\u0001J\u0004J\u0375\bJ\u000bJ\fJ\u0376\u0003"+
-    "J\u0379\bJ\u0001K\u0001K\u0001K\u0001K\u0001K\u0001L\u0001L\u0001L\u0001"+
-    "L\u0001L\u0001M\u0001M\u0005M\u0387\bM\nM\fM\u038a\tM\u0001M\u0001M\u0003"+
-    "M\u038e\bM\u0001M\u0004M\u0391\bM\u000bM\fM\u0392\u0003M\u0395\bM\u0001"+
-    "N\u0001N\u0004N\u0399\bN\u000bN\fN\u039a\u0001N\u0001N\u0001O\u0001O\u0001"+
-    "P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001"+
-    "R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001"+
-    "T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001"+
-    "W\u0001W\u0001W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001"+
-    "Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001"+
-    "[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001"+
-    "^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001"+
-    "_\u0003_\u03e8\b_\u0001`\u0004`\u03eb\b`\u000b`\f`\u03ec\u0001a\u0001"+
-    "a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001"+
-    "c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001"+
-    "f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001"+
-    "h\u0001h\u0001i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001"+
-    "k\u0001k\u0001k\u0001k\u0003k\u041c\bk\u0001l\u0001l\u0003l\u0420\bl\u0001"+
-    "l\u0005l\u0423\bl\nl\fl\u0426\tl\u0001l\u0001l\u0003l\u042a\bl\u0001l"+
-    "\u0004l\u042d\bl\u000bl\fl\u042e\u0003l\u0431\bl\u0001m\u0001m\u0004m"+
-    "\u0435\bm\u000bm\fm\u0436\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001"+
-    "o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001"+
-    "q\u0001r\u0001r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001"+
-    "t\u0001t\u0001t\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001"+
+    "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+
+    "\u0001\u0013\u0004\u0013\u0242\b\u0013\u000b\u0013\f\u0013\u0243\u0001"+
+    "\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0005"+
+    "\u0014\u024c\b\u0014\n\u0014\f\u0014\u024f\t\u0014\u0001\u0014\u0003\u0014"+
+    "\u0252\b\u0014\u0001\u0014\u0003\u0014\u0255\b\u0014\u0001\u0014\u0001"+
+    "\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005"+
+    "\u0015\u025e\b\u0015\n\u0015\f\u0015\u0261\t\u0015\u0001\u0015\u0001\u0015"+
+    "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0004\u0016\u0269\b\u0016"+
+    "\u000b\u0016\f\u0016\u026a\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017"+
+    "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019"+
+    "\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001c"+
+    "\u0001\u001c\u0003\u001c\u027e\b\u001c\u0001\u001c\u0004\u001c\u0281\b"+
+    "\u001c\u000b\u001c\f\u001c\u0282\u0001\u001d\u0001\u001d\u0001\u001e\u0001"+
+    "\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u028c\b\u001f\u0001"+
+    " \u0001 \u0001!\u0001!\u0001!\u0003!\u0293\b!\u0001\"\u0001\"\u0001\""+
+    "\u0005\"\u0298\b\"\n\"\f\"\u029b\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+
+    "\"\u0001\"\u0005\"\u02a3\b\"\n\"\f\"\u02a6\t\"\u0001\"\u0001\"\u0001\""+
+    "\u0001\"\u0001\"\u0003\"\u02ad\b\"\u0001\"\u0003\"\u02b0\b\"\u0003\"\u02b2"+
+    "\b\"\u0001#\u0004#\u02b5\b#\u000b#\f#\u02b6\u0001$\u0004$\u02ba\b$\u000b"+
+    "$\f$\u02bb\u0001$\u0001$\u0005$\u02c0\b$\n$\f$\u02c3\t$\u0001$\u0001$"+
+    "\u0004$\u02c7\b$\u000b$\f$\u02c8\u0001$\u0004$\u02cc\b$\u000b$\f$\u02cd"+
+    "\u0001$\u0001$\u0005$\u02d2\b$\n$\f$\u02d5\t$\u0003$\u02d7\b$\u0001$\u0001"+
+    "$\u0001$\u0001$\u0004$\u02dd\b$\u000b$\f$\u02de\u0001$\u0001$\u0003$\u02e3"+
+    "\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001"+
+    "\'\u0001\'\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001+\u0001"+
+    "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001"+
+    "-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001"+
+    "/\u00010\u00010\u00010\u00011\u00011\u00011\u00011\u00011\u00012\u0001"+
+    "2\u00012\u00012\u00012\u00013\u00013\u00014\u00014\u00014\u00014\u0001"+
+    "5\u00015\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00016\u0001"+
+    "6\u00017\u00017\u00017\u00018\u00018\u00019\u00019\u00019\u00019\u0001"+
+    "9\u00019\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001"+
+    "<\u0001<\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001"+
+    "@\u0001@\u0001@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001"+
+    "D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001"+
+    "H\u0001H\u0001H\u0001H\u0001I\u0001I\u0001I\u0001I\u0001J\u0001J\u0001"+
+    "J\u0003J\u0367\bJ\u0001J\u0005J\u036a\bJ\nJ\fJ\u036d\tJ\u0001J\u0001J"+
+    "\u0004J\u0371\bJ\u000bJ\fJ\u0372\u0003J\u0375\bJ\u0001K\u0001K\u0001K"+
+    "\u0001K\u0001K\u0001L\u0001L\u0001L\u0001L\u0001L\u0001M\u0001M\u0005"+
+    "M\u0383\bM\nM\fM\u0386\tM\u0001M\u0001M\u0003M\u038a\bM\u0001M\u0004M"+
+    "\u038d\bM\u000bM\fM\u038e\u0003M\u0391\bM\u0001N\u0001N\u0004N\u0395\b"+
+    "N\u000bN\fN\u0396\u0001N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001P\u0001"+
+    "P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001S\u0001"+
+    "S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001T\u0001T\u0001U\u0001"+
+    "U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001"+
+    "W\u0001X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001"+
+    "Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001"+
+    "\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001"+
+    "^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0003_\u03e4\b_\u0001"+
+    "`\u0004`\u03e7\b`\u000b`\f`\u03e8\u0001a\u0001a\u0001a\u0001a\u0001b\u0001"+
+    "b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001"+
+    "d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001f\u0001"+
+    "g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001"+
+    "i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001"+
+    "k\u0001k\u0003k\u041a\bk\u0001l\u0001l\u0003l\u041e\bl\u0001l\u0005l\u0421"+
+    "\bl\nl\fl\u0424\tl\u0001l\u0001l\u0003l\u0428\bl\u0001l\u0004l\u042b\b"+
+    "l\u000bl\fl\u042c\u0003l\u042f\bl\u0001m\u0001m\u0004m\u0433\bm\u000b"+
+    "m\fm\u0434\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001"+
+    "p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001q\u0001r\u0001"+
+    "r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001"+
+    "t\u0001u\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001"+
     "v\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+
     "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+
     "|\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001}\u0001"+
@@ -428,807 +482,809 @@ private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) {
     "\u0088\u0001\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001"+
     "\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001"+
     "\u008b\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+
-    "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001"+
-    "\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001"+
-    "\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001"+
-    "\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001"+
-    "\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001"+
-    "\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001"+
-    "\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001"+
-    "\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001"+
-    "\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001"+
-    "\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001"+
-    "\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001"+
-    "\u009c\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001"+
-    "\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001"+
-    "\u009f\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001"+
-    "\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001"+
-    "\u00a2\u0001\u00a2\u0004\u00a2\u0520\b\u00a2\u000b\u00a2\f\u00a2\u0521"+
-    "\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4"+
-    "\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5"+
-    "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7"+
-    "\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8"+
-    "\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa"+
-    "\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab"+
-    "\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac"+
-    "\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae"+
-    "\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af"+
-    "\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0"+
-    "\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2"+
-    "\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3"+
-    "\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5"+
-    "\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6"+
-    "\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8"+
-    "\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9"+
-    "\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba"+
-    "\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb"+
-    "\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd"+
-    "\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be"+
-    "\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf"+
-    "\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0"+
-    "\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2"+
-    "\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3"+
-    "\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4"+
-    "\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5"+
-    "\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0002\u0268\u02ad\u0000"+
-    "\u00c6\u000f\u0001\u0011\u0002\u0013\u0003\u0015\u0004\u0017\u0005\u0019"+
-    "\u0006\u001b\u0007\u001d\b\u001f\t!\n#\u000b%\f\'\r)\u000e+\u000f-\u0010"+
-    "/\u00111\u00123\u00135\u00147\u00159\u0016;\u0017=\u0018?\u0019A\u0000"+
-    "C\u0000E\u0000G\u0000I\u0000K\u0000M\u0000O\u0000Q\u0000S\u0000U\u001a"+
-    "W\u001bY\u001c[\u001d]\u001e_\u001fa c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f"+
-    "/\u00810\u00831\u00852\u00873\u00894\u008b5\u008d6\u008f7\u00918\u0093"+
-    "9\u0095:\u0097;\u0099<\u009b=\u009d>\u009f?\u00a1\u0000\u00a3@\u00a5A"+
-    "\u00a7B\u00a9C\u00ab\u0000\u00adD\u00afE\u00b1F\u00b3G\u00b5\u0000\u00b7"+
-    "\u0000\u00b9H\u00bbI\u00bdJ\u00bf\u0000\u00c1\u0000\u00c3\u0000\u00c5"+
-    "\u0000\u00c7\u0000\u00c9\u0000\u00cbK\u00cd\u0000\u00cfL\u00d1\u0000\u00d3"+
-    "\u0000\u00d5M\u00d7N\u00d9O\u00db\u0000\u00dd\u0000\u00df\u0000\u00e1"+
-    "\u0000\u00e3\u0000\u00e5\u0000\u00e7\u0000\u00e9P\u00ebQ\u00edR\u00ef"+
-    "S\u00f1\u0000\u00f3\u0000\u00f5\u0000\u00f7\u0000\u00f9\u0000\u00fb\u0000"+
-    "\u00fdT\u00ff\u0000\u0101U\u0103V\u0105W\u0107\u0000\u0109\u0000\u010b"+
-    "X\u010dY\u010f\u0000\u0111Z\u0113\u0000\u0115[\u0117\\\u0119]\u011b\u0000"+
-    "\u011d\u0000\u011f\u0000\u0121\u0000\u0123\u0000\u0125\u0000\u0127\u0000"+
-    "\u0129\u0000\u012b\u0000\u012d^\u012f_\u0131`\u0133\u0000\u0135\u0000"+
-    "\u0137\u0000\u0139\u0000\u013b\u0000\u013d\u0000\u013fa\u0141b\u0143c"+
-    "\u0145\u0000\u0147d\u0149e\u014bf\u014dg\u014f\u0000\u0151h\u0153i\u0155"+
-    "j\u0157k\u0159l\u015b\u0000\u015d\u0000\u015f\u0000\u0161\u0000\u0163"+
-    "\u0000\u0165\u0000\u0167\u0000\u0169m\u016bn\u016do\u016f\u0000\u0171"+
-    "\u0000\u0173\u0000\u0175\u0000\u0177p\u0179q\u017br\u017d\u0000\u017f"+
-    "\u0000\u0181\u0000\u0183s\u0185t\u0187u\u0189\u0000\u018b\u0000\u018d"+
-    "v\u018fw\u0191x\u0193\u0000\u0195\u0000\u0197\u0000\u0199\u0000\u000f"+
-    "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+
-    "#\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002"+
-    "\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000"+
-    "PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002"+
-    "\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000"+
-    "GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r"+
-    "\r  //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r  \u0001\u000009\u0002"+
-    "\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002"+
-    "\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t"+
-    "\n\r\r  \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r  \"#,,"+
-    "//::<<>?\\\\||\u05df\u0000\u000f\u0001\u0000\u0000\u0000\u0000\u0011\u0001"+
-    "\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000\u0000\u0015\u0001"+
-    "\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000\u0000\u0019\u0001"+
-    "\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000\u0000\u001d\u0001"+
-    "\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000\u0000!\u0001\u0000"+
-    "\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%\u0001\u0000\u0000\u0000"+
-    "\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001\u0000\u0000\u0000\u0000"+
-    "+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000\u0000\u0000/\u0001"+
-    "\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u00003\u0001\u0000\u0000"+
-    "\u0000\u00005\u0001\u0000\u0000\u0000\u00007\u0001\u0000\u0000\u0000\u0000"+
-    "9\u0001\u0000\u0000\u0000\u0000;\u0001\u0000\u0000\u0000\u0000=\u0001"+
-    "\u0000\u0000\u0000\u0001?\u0001\u0000\u0000\u0000\u0001U\u0001\u0000\u0000"+
-    "\u0000\u0001W\u0001\u0000\u0000\u0000\u0001Y\u0001\u0000\u0000\u0000\u0001"+
-    "[\u0001\u0000\u0000\u0000\u0001]\u0001\u0000\u0000\u0000\u0001_\u0001"+
-    "\u0000\u0000\u0000\u0001a\u0001\u0000\u0000\u0000\u0001c\u0001\u0000\u0000"+
-    "\u0000\u0001e\u0001\u0000\u0000\u0000\u0001g\u0001\u0000\u0000\u0000\u0001"+
-    "i\u0001\u0000\u0000\u0000\u0001k\u0001\u0000\u0000\u0000\u0001m\u0001"+
-    "\u0000\u0000\u0000\u0001o\u0001\u0000\u0000\u0000\u0001q\u0001\u0000\u0000"+
-    "\u0000\u0001s\u0001\u0000\u0000\u0000\u0001u\u0001\u0000\u0000\u0000\u0001"+
-    "w\u0001\u0000\u0000\u0000\u0001y\u0001\u0000\u0000\u0000\u0001{\u0001"+
-    "\u0000\u0000\u0000\u0001}\u0001\u0000\u0000\u0000\u0001\u007f\u0001\u0000"+
-    "\u0000\u0000\u0001\u0081\u0001\u0000\u0000\u0000\u0001\u0083\u0001\u0000"+
-    "\u0000\u0000\u0001\u0085\u0001\u0000\u0000\u0000\u0001\u0087\u0001\u0000"+
-    "\u0000\u0000\u0001\u0089\u0001\u0000\u0000\u0000\u0001\u008b\u0001\u0000"+
-    "\u0000\u0000\u0001\u008d\u0001\u0000\u0000\u0000\u0001\u008f\u0001\u0000"+
-    "\u0000\u0000\u0001\u0091\u0001\u0000\u0000\u0000\u0001\u0093\u0001\u0000"+
-    "\u0000\u0000\u0001\u0095\u0001\u0000\u0000\u0000\u0001\u0097\u0001\u0000"+
-    "\u0000\u0000\u0001\u0099\u0001\u0000\u0000\u0000\u0001\u009b\u0001\u0000"+
-    "\u0000\u0000\u0001\u009d\u0001\u0000\u0000\u0000\u0001\u009f\u0001\u0000"+
-    "\u0000\u0000\u0001\u00a1\u0001\u0000\u0000\u0000\u0001\u00a3\u0001\u0000"+
-    "\u0000\u0000\u0001\u00a5\u0001\u0000\u0000\u0000\u0001\u00a7\u0001\u0000"+
-    "\u0000\u0000\u0001\u00a9\u0001\u0000\u0000\u0000\u0001\u00ad\u0001\u0000"+
-    "\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001\u00b1\u0001\u0000"+
-    "\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0002\u00b5\u0001\u0000"+
-    "\u0000\u0000\u0002\u00b7\u0001\u0000\u0000\u0000\u0002\u00b9\u0001\u0000"+
-    "\u0000\u0000\u0002\u00bb\u0001\u0000\u0000\u0000\u0002\u00bd\u0001\u0000"+
-    "\u0000\u0000\u0003\u00bf\u0001\u0000\u0000\u0000\u0003\u00c1\u0001\u0000"+
-    "\u0000\u0000\u0003\u00c3\u0001\u0000\u0000\u0000\u0003\u00c5\u0001\u0000"+
-    "\u0000\u0000\u0003\u00c7\u0001\u0000\u0000\u0000\u0003\u00c9\u0001\u0000"+
-    "\u0000\u0000\u0003\u00cb\u0001\u0000\u0000\u0000\u0003\u00cf\u0001\u0000"+
-    "\u0000\u0000\u0003\u00d1\u0001\u0000\u0000\u0000\u0003\u00d3\u0001\u0000"+
-    "\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003\u00d7\u0001\u0000"+
-    "\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0004\u00db\u0001\u0000"+
-    "\u0000\u0000\u0004\u00dd\u0001\u0000\u0000\u0000\u0004\u00df\u0001\u0000"+
-    "\u0000\u0000\u0004\u00e1\u0001\u0000\u0000\u0000\u0004\u00e3\u0001\u0000"+
-    "\u0000\u0000\u0004\u00e9\u0001\u0000\u0000\u0000\u0004\u00eb\u0001\u0000"+
-    "\u0000\u0000\u0004\u00ed\u0001\u0000\u0000\u0000\u0004\u00ef\u0001\u0000"+
-    "\u0000\u0000\u0005\u00f1\u0001\u0000\u0000\u0000\u0005\u00f3\u0001\u0000"+
-    "\u0000\u0000\u0005\u00f5\u0001\u0000\u0000\u0000\u0005\u00f7\u0001\u0000"+
-    "\u0000\u0000\u0005\u00f9\u0001\u0000\u0000\u0000\u0005\u00fb\u0001\u0000"+
-    "\u0000\u0000\u0005\u00fd\u0001\u0000\u0000\u0000\u0005\u00ff\u0001\u0000"+
-    "\u0000\u0000\u0005\u0101\u0001\u0000\u0000\u0000\u0005\u0103\u0001\u0000"+
-    "\u0000\u0000\u0005\u0105\u0001\u0000\u0000\u0000\u0006\u0107\u0001\u0000"+
-    "\u0000\u0000\u0006\u0109\u0001\u0000\u0000\u0000\u0006\u010b\u0001\u0000"+
-    "\u0000\u0000\u0006\u010d\u0001\u0000\u0000\u0000\u0006\u0111\u0001\u0000"+
-    "\u0000\u0000\u0006\u0113\u0001\u0000\u0000\u0000\u0006\u0115\u0001\u0000"+
-    "\u0000\u0000\u0006\u0117\u0001\u0000\u0000\u0000\u0006\u0119\u0001\u0000"+
-    "\u0000\u0000\u0007\u011b\u0001\u0000\u0000\u0000\u0007\u011d\u0001\u0000"+
-    "\u0000\u0000\u0007\u011f\u0001\u0000\u0000\u0000\u0007\u0121\u0001\u0000"+
-    "\u0000\u0000\u0007\u0123\u0001\u0000\u0000\u0000\u0007\u0125\u0001\u0000"+
-    "\u0000\u0000\u0007\u0127\u0001\u0000\u0000\u0000\u0007\u0129\u0001\u0000"+
-    "\u0000\u0000\u0007\u012b\u0001\u0000\u0000\u0000\u0007\u012d\u0001\u0000"+
-    "\u0000\u0000\u0007\u012f\u0001\u0000\u0000\u0000\u0007\u0131\u0001\u0000"+
-    "\u0000\u0000\b\u0133\u0001\u0000\u0000\u0000\b\u0135\u0001\u0000\u0000"+
-    "\u0000\b\u0137\u0001\u0000\u0000\u0000\b\u0139\u0001\u0000\u0000\u0000"+
-    "\b\u013b\u0001\u0000\u0000\u0000\b\u013d\u0001\u0000\u0000\u0000\b\u013f"+
-    "\u0001\u0000\u0000\u0000\b\u0141\u0001\u0000\u0000\u0000\b\u0143\u0001"+
-    "\u0000\u0000\u0000\t\u0145\u0001\u0000\u0000\u0000\t\u0147\u0001\u0000"+
-    "\u0000\u0000\t\u0149\u0001\u0000\u0000\u0000\t\u014b\u0001\u0000\u0000"+
-    "\u0000\t\u014d\u0001\u0000\u0000\u0000\n\u014f\u0001\u0000\u0000\u0000"+
-    "\n\u0151\u0001\u0000\u0000\u0000\n\u0153\u0001\u0000\u0000\u0000\n\u0155"+
-    "\u0001\u0000\u0000\u0000\n\u0157\u0001\u0000\u0000\u0000\n\u0159\u0001"+
-    "\u0000\u0000\u0000\u000b\u015b\u0001\u0000\u0000\u0000\u000b\u015d\u0001"+
-    "\u0000\u0000\u0000\u000b\u015f\u0001\u0000\u0000\u0000\u000b\u0161\u0001"+
-    "\u0000\u0000\u0000\u000b\u0163\u0001\u0000\u0000\u0000\u000b\u0165\u0001"+
-    "\u0000\u0000\u0000\u000b\u0167\u0001\u0000\u0000\u0000\u000b\u0169\u0001"+
-    "\u0000\u0000\u0000\u000b\u016b\u0001\u0000\u0000\u0000\u000b\u016d\u0001"+
-    "\u0000\u0000\u0000\f\u016f\u0001\u0000\u0000\u0000\f\u0171\u0001\u0000"+
-    "\u0000\u0000\f\u0173\u0001\u0000\u0000\u0000\f\u0175\u0001\u0000\u0000"+
-    "\u0000\f\u0177\u0001\u0000\u0000\u0000\f\u0179\u0001\u0000\u0000\u0000"+
-    "\f\u017b\u0001\u0000\u0000\u0000\r\u017d\u0001\u0000\u0000\u0000\r\u017f"+
-    "\u0001\u0000\u0000\u0000\r\u0181\u0001\u0000\u0000\u0000\r\u0183\u0001"+
-    "\u0000\u0000\u0000\r\u0185\u0001\u0000\u0000\u0000\r\u0187\u0001\u0000"+
-    "\u0000\u0000\u000e\u0189\u0001\u0000\u0000\u0000\u000e\u018b\u0001\u0000"+
-    "\u0000\u0000\u000e\u018d\u0001\u0000\u0000\u0000\u000e\u018f\u0001\u0000"+
-    "\u0000\u0000\u000e\u0191\u0001\u0000\u0000\u0000\u000e\u0193\u0001\u0000"+
-    "\u0000\u0000\u000e\u0195\u0001\u0000\u0000\u0000\u000e\u0197\u0001\u0000"+
-    "\u0000\u0000\u000e\u0199\u0001\u0000\u0000\u0000\u000f\u019b\u0001\u0000"+
-    "\u0000\u0000\u0011\u01a5\u0001\u0000\u0000\u0000\u0013\u01ac\u0001\u0000"+
-    "\u0000\u0000\u0015\u01b5\u0001\u0000\u0000\u0000\u0017\u01bc\u0001\u0000"+
-    "\u0000\u0000\u0019\u01c6\u0001\u0000\u0000\u0000\u001b\u01cd\u0001\u0000"+
-    "\u0000\u0000\u001d\u01d4\u0001\u0000\u0000\u0000\u001f\u01db\u0001\u0000"+
-    "\u0000\u0000!\u01e3\u0001\u0000\u0000\u0000#\u01ef\u0001\u0000\u0000\u0000"+
-    "%\u01f8\u0001\u0000\u0000\u0000\'\u01fe\u0001\u0000\u0000\u0000)\u0205"+
-    "\u0001\u0000\u0000\u0000+\u020c\u0001\u0000\u0000\u0000-\u0214\u0001\u0000"+
-    "\u0000\u0000/\u021c\u0001\u0000\u0000\u00001\u022b\u0001\u0000\u0000\u0000"+
-    "3\u0235\u0001\u0000\u0000\u00005\u023e\u0001\u0000\u0000\u00007\u024a"+
-    "\u0001\u0000\u0000\u00009\u0250\u0001\u0000\u0000\u0000;\u0261\u0001\u0000"+
-    "\u0000\u0000=\u0271\u0001\u0000\u0000\u0000?\u0277\u0001\u0000\u0000\u0000"+
-    "A\u027b\u0001\u0000\u0000\u0000C\u027d\u0001\u0000\u0000\u0000E\u027f"+
-    "\u0001\u0000\u0000\u0000G\u0282\u0001\u0000\u0000\u0000I\u0284\u0001\u0000"+
-    "\u0000\u0000K\u028d\u0001\u0000\u0000\u0000M\u028f\u0001\u0000\u0000\u0000"+
-    "O\u0294\u0001\u0000\u0000\u0000Q\u0296\u0001\u0000\u0000\u0000S\u029b"+
-    "\u0001\u0000\u0000\u0000U\u02ba\u0001\u0000\u0000\u0000W\u02bd\u0001\u0000"+
-    "\u0000\u0000Y\u02eb\u0001\u0000\u0000\u0000[\u02ed\u0001\u0000\u0000\u0000"+
-    "]\u02f0\u0001\u0000\u0000\u0000_\u02f4\u0001\u0000\u0000\u0000a\u02f8"+
-    "\u0001\u0000\u0000\u0000c\u02fa\u0001\u0000\u0000\u0000e\u02fd\u0001\u0000"+
-    "\u0000\u0000g\u02ff\u0001\u0000\u0000\u0000i\u0304\u0001\u0000\u0000\u0000"+
-    "k\u0306\u0001\u0000\u0000\u0000m\u030c\u0001\u0000\u0000\u0000o\u0312"+
-    "\u0001\u0000\u0000\u0000q\u0315\u0001\u0000\u0000\u0000s\u0318\u0001\u0000"+
-    "\u0000\u0000u\u031d\u0001\u0000\u0000\u0000w\u0322\u0001\u0000\u0000\u0000"+
-    "y\u0324\u0001\u0000\u0000\u0000{\u0328\u0001\u0000\u0000\u0000}\u032d"+
-    "\u0001\u0000\u0000\u0000\u007f\u0333\u0001\u0000\u0000\u0000\u0081\u0336"+
-    "\u0001\u0000\u0000\u0000\u0083\u0338\u0001\u0000\u0000\u0000\u0085\u033e"+
-    "\u0001\u0000\u0000\u0000\u0087\u0340\u0001\u0000\u0000\u0000\u0089\u0345"+
-    "\u0001\u0000\u0000\u0000\u008b\u0348\u0001\u0000\u0000\u0000\u008d\u034b"+
-    "\u0001\u0000\u0000\u0000\u008f\u034e\u0001\u0000\u0000\u0000\u0091\u0350"+
-    "\u0001\u0000\u0000\u0000\u0093\u0353\u0001\u0000\u0000\u0000\u0095\u0355"+
-    "\u0001\u0000\u0000\u0000\u0097\u0358\u0001\u0000\u0000\u0000\u0099\u035a"+
-    "\u0001\u0000\u0000\u0000\u009b\u035c\u0001\u0000\u0000\u0000\u009d\u035e"+
-    "\u0001\u0000\u0000\u0000\u009f\u0360\u0001\u0000\u0000\u0000\u00a1\u0362"+
-    "\u0001\u0000\u0000\u0000\u00a3\u0378\u0001\u0000\u0000\u0000\u00a5\u037a"+
-    "\u0001\u0000\u0000\u0000\u00a7\u037f\u0001\u0000\u0000\u0000\u00a9\u0394"+
-    "\u0001\u0000\u0000\u0000\u00ab\u0396\u0001\u0000\u0000\u0000\u00ad\u039e"+
-    "\u0001\u0000\u0000\u0000\u00af\u03a0\u0001\u0000\u0000\u0000\u00b1\u03a4"+
-    "\u0001\u0000\u0000\u0000\u00b3\u03a8\u0001\u0000\u0000\u0000\u00b5\u03ac"+
-    "\u0001\u0000\u0000\u0000\u00b7\u03b1\u0001\u0000\u0000\u0000\u00b9\u03b6"+
-    "\u0001\u0000\u0000\u0000\u00bb\u03ba\u0001\u0000\u0000\u0000\u00bd\u03be"+
-    "\u0001\u0000\u0000\u0000\u00bf\u03c2\u0001\u0000\u0000\u0000\u00c1\u03c7"+
-    "\u0001\u0000\u0000\u0000\u00c3\u03cb\u0001\u0000\u0000\u0000\u00c5\u03cf"+
-    "\u0001\u0000\u0000\u0000\u00c7\u03d3\u0001\u0000\u0000\u0000\u00c9\u03d7"+
-    "\u0001\u0000\u0000\u0000\u00cb\u03db\u0001\u0000\u0000\u0000\u00cd\u03e7"+
-    "\u0001\u0000\u0000\u0000\u00cf\u03ea\u0001\u0000\u0000\u0000\u00d1\u03ee"+
-    "\u0001\u0000\u0000\u0000\u00d3\u03f2\u0001\u0000\u0000\u0000\u00d5\u03f6"+
-    "\u0001\u0000\u0000\u0000\u00d7\u03fa\u0001\u0000\u0000\u0000\u00d9\u03fe"+
-    "\u0001\u0000\u0000\u0000\u00db\u0402\u0001\u0000\u0000\u0000\u00dd\u0407"+
-    "\u0001\u0000\u0000\u0000\u00df\u040b\u0001\u0000\u0000\u0000\u00e1\u040f"+
-    "\u0001\u0000\u0000\u0000\u00e3\u0413\u0001\u0000\u0000\u0000\u00e5\u041b"+
-    "\u0001\u0000\u0000\u0000\u00e7\u0430\u0001\u0000\u0000\u0000\u00e9\u0434"+
-    "\u0001\u0000\u0000\u0000\u00eb\u0438\u0001\u0000\u0000\u0000\u00ed\u043c"+
-    "\u0001\u0000\u0000\u0000\u00ef\u0440\u0001\u0000\u0000\u0000\u00f1\u0444"+
-    "\u0001\u0000\u0000\u0000\u00f3\u0449\u0001\u0000\u0000\u0000\u00f5\u044d"+
-    "\u0001\u0000\u0000\u0000\u00f7\u0451\u0001\u0000\u0000\u0000\u00f9\u0455"+
-    "\u0001\u0000\u0000\u0000\u00fb\u0459\u0001\u0000\u0000\u0000\u00fd\u045d"+
-    "\u0001\u0000\u0000\u0000\u00ff\u0460\u0001\u0000\u0000\u0000\u0101\u0464"+
-    "\u0001\u0000\u0000\u0000\u0103\u0468\u0001\u0000\u0000\u0000\u0105\u046c"+
-    "\u0001\u0000\u0000\u0000\u0107\u0470\u0001\u0000\u0000\u0000\u0109\u0475"+
-    "\u0001\u0000\u0000\u0000\u010b\u047a\u0001\u0000\u0000\u0000\u010d\u047f"+
-    "\u0001\u0000\u0000\u0000\u010f\u0486\u0001\u0000\u0000\u0000\u0111\u048f"+
-    "\u0001\u0000\u0000\u0000\u0113\u0496\u0001\u0000\u0000\u0000\u0115\u049a"+
-    "\u0001\u0000\u0000\u0000\u0117\u049e\u0001\u0000\u0000\u0000\u0119\u04a2"+
-    "\u0001\u0000\u0000\u0000\u011b\u04a6\u0001\u0000\u0000\u0000\u011d\u04ac"+
-    "\u0001\u0000\u0000\u0000\u011f\u04b0\u0001\u0000\u0000\u0000\u0121\u04b4"+
-    "\u0001\u0000\u0000\u0000\u0123\u04b8\u0001\u0000\u0000\u0000\u0125\u04bc"+
-    "\u0001\u0000\u0000\u0000\u0127\u04c0\u0001\u0000\u0000\u0000\u0129\u04c4"+
-    "\u0001\u0000\u0000\u0000\u012b\u04c8\u0001\u0000\u0000\u0000\u012d\u04cc"+
-    "\u0001\u0000\u0000\u0000\u012f\u04d0\u0001\u0000\u0000\u0000\u0131\u04d4"+
-    "\u0001\u0000\u0000\u0000\u0133\u04d8\u0001\u0000\u0000\u0000\u0135\u04dd"+
-    "\u0001\u0000\u0000\u0000\u0137\u04e1\u0001\u0000\u0000\u0000\u0139\u04e5"+
-    "\u0001\u0000\u0000\u0000\u013b\u04e9\u0001\u0000\u0000\u0000\u013d\u04ed"+
-    "\u0001\u0000\u0000\u0000\u013f\u04f1\u0001\u0000\u0000\u0000\u0141\u04f5"+
-    "\u0001\u0000\u0000\u0000\u0143\u04f9\u0001\u0000\u0000\u0000\u0145\u04fd"+
-    "\u0001\u0000\u0000\u0000\u0147\u0502\u0001\u0000\u0000\u0000\u0149\u0507"+
-    "\u0001\u0000\u0000\u0000\u014b\u050b\u0001\u0000\u0000\u0000\u014d\u050f"+
-    "\u0001\u0000\u0000\u0000\u014f\u0513\u0001\u0000\u0000\u0000\u0151\u0518"+
-    "\u0001\u0000\u0000\u0000\u0153\u051f\u0001\u0000\u0000\u0000\u0155\u0523"+
-    "\u0001\u0000\u0000\u0000\u0157\u0527\u0001\u0000\u0000\u0000\u0159\u052b"+
-    "\u0001\u0000\u0000\u0000\u015b\u052f\u0001\u0000\u0000\u0000\u015d\u0534"+
-    "\u0001\u0000\u0000\u0000\u015f\u0538\u0001\u0000\u0000\u0000\u0161\u053c"+
-    "\u0001\u0000\u0000\u0000\u0163\u0540\u0001\u0000\u0000\u0000\u0165\u0545"+
-    "\u0001\u0000\u0000\u0000\u0167\u0549\u0001\u0000\u0000\u0000\u0169\u054d"+
-    "\u0001\u0000\u0000\u0000\u016b\u0551\u0001\u0000\u0000\u0000\u016d\u0555"+
-    "\u0001\u0000\u0000\u0000\u016f\u0559\u0001\u0000\u0000\u0000\u0171\u055f"+
-    "\u0001\u0000\u0000\u0000\u0173\u0563\u0001\u0000\u0000\u0000\u0175\u0567"+
-    "\u0001\u0000\u0000\u0000\u0177\u056b\u0001\u0000\u0000\u0000\u0179\u056f"+
-    "\u0001\u0000\u0000\u0000\u017b\u0573\u0001\u0000\u0000\u0000\u017d\u0577"+
-    "\u0001\u0000\u0000\u0000\u017f\u057c\u0001\u0000\u0000\u0000\u0181\u0582"+
-    "\u0001\u0000\u0000\u0000\u0183\u0588\u0001\u0000\u0000\u0000\u0185\u058c"+
-    "\u0001\u0000\u0000\u0000\u0187\u0590\u0001\u0000\u0000\u0000\u0189\u0594"+
-    "\u0001\u0000\u0000\u0000\u018b\u059a\u0001\u0000\u0000\u0000\u018d\u05a0"+
-    "\u0001\u0000\u0000\u0000\u018f\u05a4\u0001\u0000\u0000\u0000\u0191\u05a8"+
-    "\u0001\u0000\u0000\u0000\u0193\u05ac\u0001\u0000\u0000\u0000\u0195\u05b2"+
-    "\u0001\u0000\u0000\u0000\u0197\u05b8\u0001\u0000\u0000\u0000\u0199\u05be"+
-    "\u0001\u0000\u0000\u0000\u019b\u019c\u0007\u0000\u0000\u0000\u019c\u019d"+
-    "\u0007\u0001\u0000\u0000\u019d\u019e\u0007\u0002\u0000\u0000\u019e\u019f"+
-    "\u0007\u0002\u0000\u0000\u019f\u01a0\u0007\u0003\u0000\u0000\u01a0\u01a1"+
-    "\u0007\u0004\u0000\u0000\u01a1\u01a2\u0007\u0005\u0000\u0000\u01a2\u01a3"+
-    "\u0001\u0000\u0000\u0000\u01a3\u01a4\u0006\u0000\u0000\u0000\u01a4\u0010"+
-    "\u0001\u0000\u0000\u0000\u01a5\u01a6\u0007\u0000\u0000\u0000\u01a6\u01a7"+
-    "\u0007\u0006\u0000\u0000\u01a7\u01a8\u0007\u0007\u0000\u0000\u01a8\u01a9"+
-    "\u0007\b\u0000\u0000\u01a9\u01aa\u0001\u0000\u0000\u0000\u01aa\u01ab\u0006"+
-    "\u0001\u0001\u0000\u01ab\u0012\u0001\u0000\u0000\u0000\u01ac\u01ad\u0007"+
-    "\u0003\u0000\u0000\u01ad\u01ae\u0007\t\u0000\u0000\u01ae\u01af\u0007\u0006"+
-    "\u0000\u0000\u01af\u01b0\u0007\u0001\u0000\u0000\u01b0\u01b1\u0007\u0004"+
-    "\u0000\u0000\u01b1\u01b2\u0007\n\u0000\u0000\u01b2\u01b3\u0001\u0000\u0000"+
-    "\u0000\u01b3\u01b4\u0006\u0002\u0002\u0000\u01b4\u0014\u0001\u0000\u0000"+
-    "\u0000\u01b5\u01b6\u0007\u0003\u0000\u0000\u01b6\u01b7\u0007\u000b\u0000"+
-    "\u0000\u01b7\u01b8\u0007\f\u0000\u0000\u01b8\u01b9\u0007\r\u0000\u0000"+
-    "\u01b9\u01ba\u0001\u0000\u0000\u0000\u01ba\u01bb\u0006\u0003\u0000\u0000"+
-    "\u01bb\u0016\u0001\u0000\u0000\u0000\u01bc\u01bd\u0007\u0003\u0000\u0000"+
-    "\u01bd\u01be\u0007\u000e\u0000\u0000\u01be\u01bf\u0007\b\u0000\u0000\u01bf"+
-    "\u01c0\u0007\r\u0000\u0000\u01c0\u01c1\u0007\f\u0000\u0000\u01c1\u01c2"+
-    "\u0007\u0001\u0000\u0000\u01c2\u01c3\u0007\t\u0000\u0000\u01c3\u01c4\u0001"+
-    "\u0000\u0000\u0000\u01c4\u01c5\u0006\u0004\u0003\u0000\u01c5\u0018\u0001"+
-    "\u0000\u0000\u0000\u01c6\u01c7\u0007\u000f\u0000\u0000\u01c7\u01c8\u0007"+
-    "\u0006\u0000\u0000\u01c8\u01c9\u0007\u0007\u0000\u0000\u01c9\u01ca\u0007"+
-    "\u0010\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cc\u0006"+
-    "\u0005\u0004\u0000\u01cc\u001a\u0001\u0000\u0000\u0000\u01cd\u01ce\u0007"+
-    "\u0011\u0000\u0000\u01ce\u01cf\u0007\u0006\u0000\u0000\u01cf\u01d0\u0007"+
-    "\u0007\u0000\u0000\u01d0\u01d1\u0007\u0012\u0000\u0000\u01d1\u01d2\u0001"+
-    "\u0000\u0000\u0000\u01d2\u01d3\u0006\u0006\u0000\u0000\u01d3\u001c\u0001"+
-    "\u0000\u0000\u0000\u01d4\u01d5\u0007\u0012\u0000\u0000\u01d5\u01d6\u0007"+
-    "\u0003\u0000\u0000\u01d6\u01d7\u0007\u0003\u0000\u0000\u01d7\u01d8\u0007"+
-    "\b\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da\u0006\u0007"+
-    "\u0001\u0000\u01da\u001e\u0001\u0000\u0000\u0000\u01db\u01dc\u0007\r\u0000"+
-    "\u0000\u01dc\u01dd\u0007\u0001\u0000\u0000\u01dd\u01de\u0007\u0010\u0000"+
-    "\u0000\u01de\u01df\u0007\u0001\u0000\u0000\u01df\u01e0\u0007\u0005\u0000"+
-    "\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e1\u01e2\u0006\b\u0000\u0000"+
-    "\u01e2 \u0001\u0000\u0000\u0000\u01e3\u01e4\u0007\u0010\u0000\u0000\u01e4"+
-    "\u01e5\u0007\u000b\u0000\u0000\u01e5\u01e6\u0005_\u0000\u0000\u01e6\u01e7"+
-    "\u0007\u0003\u0000\u0000\u01e7\u01e8\u0007\u000e\u0000\u0000\u01e8\u01e9"+
-    "\u0007\b\u0000\u0000\u01e9\u01ea\u0007\f\u0000\u0000\u01ea\u01eb\u0007"+
-    "\t\u0000\u0000\u01eb\u01ec\u0007\u0000\u0000\u0000\u01ec\u01ed\u0001\u0000"+
-    "\u0000\u0000\u01ed\u01ee\u0006\t\u0005\u0000\u01ee\"\u0001\u0000\u0000"+
-    "\u0000\u01ef\u01f0\u0007\u0006\u0000\u0000\u01f0\u01f1\u0007\u0003\u0000"+
-    "\u0000\u01f1\u01f2\u0007\t\u0000\u0000\u01f2\u01f3\u0007\f\u0000\u0000"+
-    "\u01f3\u01f4\u0007\u0010\u0000\u0000\u01f4\u01f5\u0007\u0003\u0000\u0000"+
-    "\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0006\n\u0006\u0000\u01f7"+
-    "$\u0001\u0000\u0000\u0000\u01f8\u01f9\u0007\u0006\u0000\u0000\u01f9\u01fa"+
-    "\u0007\u0007\u0000\u0000\u01fa\u01fb\u0007\u0013\u0000\u0000\u01fb\u01fc"+
-    "\u0001\u0000\u0000\u0000\u01fc\u01fd\u0006\u000b\u0000\u0000\u01fd&\u0001"+
-    "\u0000\u0000\u0000\u01fe\u01ff\u0007\u0002\u0000\u0000\u01ff\u0200\u0007"+
-    "\n\u0000\u0000\u0200\u0201\u0007\u0007\u0000\u0000\u0201\u0202\u0007\u0013"+
-    "\u0000\u0000\u0202\u0203\u0001\u0000\u0000\u0000\u0203\u0204\u0006\f\u0007"+
-    "\u0000\u0204(\u0001\u0000\u0000\u0000\u0205\u0206\u0007\u0002\u0000\u0000"+
-    "\u0206\u0207\u0007\u0007\u0000\u0000\u0207\u0208\u0007\u0006\u0000\u0000"+
-    "\u0208\u0209\u0007\u0005\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000"+
-    "\u020a\u020b\u0006\r\u0000\u0000\u020b*\u0001\u0000\u0000\u0000\u020c"+
-    "\u020d\u0007\u0002\u0000\u0000\u020d\u020e\u0007\u0005\u0000\u0000\u020e"+
-    "\u020f\u0007\f\u0000\u0000\u020f\u0210\u0007\u0005\u0000\u0000\u0210\u0211"+
-    "\u0007\u0002\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212\u0213"+
-    "\u0006\u000e\u0000\u0000\u0213,\u0001\u0000\u0000\u0000\u0214\u0215\u0007"+
-    "\u0013\u0000\u0000\u0215\u0216\u0007\n\u0000\u0000\u0216\u0217\u0007\u0003"+
-    "\u0000\u0000\u0217\u0218\u0007\u0006\u0000\u0000\u0218\u0219\u0007\u0003"+
-    "\u0000\u0000\u0219\u021a\u0001\u0000\u0000\u0000\u021a\u021b\u0006\u000f"+
-    "\u0000\u0000\u021b.\u0001\u0000\u0000\u0000\u021c\u021d\u0004\u0010\u0000"+
-    "\u0000\u021d\u021e\u0007\u0001\u0000\u0000\u021e\u021f\u0007\t\u0000\u0000"+
-    "\u021f\u0220\u0007\r\u0000\u0000\u0220\u0221\u0007\u0001\u0000\u0000\u0221"+
-    "\u0222\u0007\t\u0000\u0000\u0222\u0223\u0007\u0003\u0000\u0000\u0223\u0224"+
-    "\u0007\u0002\u0000\u0000\u0224\u0225\u0007\u0005\u0000\u0000\u0225\u0226"+
-    "\u0007\f\u0000\u0000\u0226\u0227\u0007\u0005\u0000\u0000\u0227\u0228\u0007"+
-    "\u0002\u0000\u0000\u0228\u0229\u0001\u0000\u0000\u0000\u0229\u022a\u0006"+
-    "\u0010\u0000\u0000\u022a0\u0001\u0000\u0000\u0000\u022b\u022c\u0004\u0011"+
-    "\u0001\u0000\u022c\u022d\u0007\r\u0000\u0000\u022d\u022e\u0007\u0007\u0000"+
-    "\u0000\u022e\u022f\u0007\u0007\u0000\u0000\u022f\u0230\u0007\u0012\u0000"+
-    "\u0000\u0230\u0231\u0007\u0014\u0000\u0000\u0231\u0232\u0007\b\u0000\u0000"+
-    "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0006\u0011\b\u0000\u0234"+
-    "2\u0001\u0000\u0000\u0000\u0235\u0236\u0004\u0012\u0002\u0000\u0236\u0237"+
-    "\u0007\u0010\u0000\u0000\u0237\u0238\u0007\f\u0000\u0000\u0238\u0239\u0007"+
-    "\u0005\u0000\u0000\u0239\u023a\u0007\u0004\u0000\u0000\u023a\u023b\u0007"+
-    "\n\u0000\u0000\u023b\u023c\u0001\u0000\u0000\u0000\u023c\u023d\u0006\u0012"+
-    "\u0000\u0000\u023d4\u0001\u0000\u0000\u0000\u023e\u023f\u0004\u0013\u0003"+
-    "\u0000\u023f\u0240\u0007\u0010\u0000\u0000\u0240\u0241\u0007\u0003\u0000"+
-    "\u0000\u0241\u0242\u0007\u0005\u0000\u0000\u0242\u0243\u0007\u0006\u0000"+
-    "\u0000\u0243\u0244\u0007\u0001\u0000\u0000\u0244\u0245\u0007\u0004\u0000"+
-    "\u0000\u0245\u0246\u0007\u0002\u0000\u0000\u0246\u0247\u0001\u0000\u0000"+
-    "\u0000\u0247\u0248\u0006\u0013\t\u0000\u02486\u0001\u0000\u0000\u0000"+
-    "\u0249\u024b\b\u0015\u0000\u0000\u024a\u0249\u0001\u0000\u0000\u0000\u024b"+
-    "\u024c\u0001\u0000\u0000\u0000\u024c\u024a\u0001\u0000\u0000\u0000\u024c"+
-    "\u024d\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e"+
-    "\u024f\u0006\u0014\u0000\u0000\u024f8\u0001\u0000\u0000\u0000\u0250\u0251"+
-    "\u0005/\u0000\u0000\u0251\u0252\u0005/\u0000\u0000\u0252\u0256\u0001\u0000"+
-    "\u0000\u0000\u0253\u0255\b\u0016\u0000\u0000\u0254\u0253\u0001\u0000\u0000"+
-    "\u0000\u0255\u0258\u0001\u0000\u0000\u0000\u0256\u0254\u0001\u0000\u0000"+
-    "\u0000\u0256\u0257\u0001\u0000\u0000\u0000\u0257\u025a\u0001\u0000\u0000"+
-    "\u0000\u0258\u0256\u0001\u0000\u0000\u0000\u0259\u025b\u0005\r\u0000\u0000"+
-    "\u025a\u0259\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000\u0000"+
-    "\u025b\u025d\u0001\u0000\u0000\u0000\u025c\u025e\u0005\n\u0000\u0000\u025d"+
-    "\u025c\u0001\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000\u025e"+
-    "\u025f\u0001\u0000\u0000\u0000\u025f\u0260\u0006\u0015\n\u0000\u0260:"+
-    "\u0001\u0000\u0000\u0000\u0261\u0262\u0005/\u0000\u0000\u0262\u0263\u0005"+
-    "*\u0000\u0000\u0263\u0268\u0001\u0000\u0000\u0000\u0264\u0267\u0003;\u0016"+
-    "\u0000\u0265\u0267\t\u0000\u0000\u0000\u0266\u0264\u0001\u0000\u0000\u0000"+
-    "\u0266\u0265\u0001\u0000\u0000\u0000\u0267\u026a\u0001\u0000\u0000\u0000"+
-    "\u0268\u0269\u0001\u0000\u0000\u0000\u0268\u0266\u0001\u0000\u0000\u0000"+
-    "\u0269\u026b\u0001\u0000\u0000\u0000\u026a\u0268\u0001\u0000\u0000\u0000"+
-    "\u026b\u026c\u0005*\u0000\u0000\u026c\u026d\u0005/\u0000\u0000\u026d\u026e"+
-    "\u0001\u0000\u0000\u0000\u026e\u026f\u0006\u0016\n\u0000\u026f<\u0001"+
-    "\u0000\u0000\u0000\u0270\u0272\u0007\u0017\u0000\u0000\u0271\u0270\u0001"+
-    "\u0000\u0000\u0000\u0272\u0273\u0001\u0000\u0000\u0000\u0273\u0271\u0001"+
-    "\u0000\u0000\u0000\u0273\u0274\u0001\u0000\u0000\u0000\u0274\u0275\u0001"+
-    "\u0000\u0000\u0000\u0275\u0276\u0006\u0017\n\u0000\u0276>\u0001\u0000"+
-    "\u0000\u0000\u0277\u0278\u0005|\u0000\u0000\u0278\u0279\u0001\u0000\u0000"+
-    "\u0000\u0279\u027a\u0006\u0018\u000b\u0000\u027a@\u0001\u0000\u0000\u0000"+
-    "\u027b\u027c\u0007\u0018\u0000\u0000\u027cB\u0001\u0000\u0000\u0000\u027d"+
-    "\u027e\u0007\u0019\u0000\u0000\u027eD\u0001\u0000\u0000\u0000\u027f\u0280"+
-    "\u0005\\\u0000\u0000\u0280\u0281\u0007\u001a\u0000\u0000\u0281F\u0001"+
-    "\u0000\u0000\u0000\u0282\u0283\b\u001b\u0000\u0000\u0283H\u0001\u0000"+
-    "\u0000\u0000\u0284\u0286\u0007\u0003\u0000\u0000\u0285\u0287\u0007\u001c"+
-    "\u0000\u0000\u0286\u0285\u0001\u0000\u0000\u0000\u0286\u0287\u0001\u0000"+
-    "\u0000\u0000\u0287\u0289\u0001\u0000\u0000\u0000\u0288\u028a\u0003A\u0019"+
-    "\u0000\u0289\u0288\u0001\u0000\u0000\u0000\u028a\u028b\u0001\u0000\u0000"+
-    "\u0000\u028b\u0289\u0001\u0000\u0000\u0000\u028b\u028c\u0001\u0000\u0000"+
-    "\u0000\u028cJ\u0001\u0000\u0000\u0000\u028d\u028e\u0005@\u0000\u0000\u028e"+
-    "L\u0001\u0000\u0000\u0000\u028f\u0290\u0005`\u0000\u0000\u0290N\u0001"+
-    "\u0000\u0000\u0000\u0291\u0295\b\u001d\u0000\u0000\u0292\u0293\u0005`"+
-    "\u0000\u0000\u0293\u0295\u0005`\u0000\u0000\u0294\u0291\u0001\u0000\u0000"+
-    "\u0000\u0294\u0292\u0001\u0000\u0000\u0000\u0295P\u0001\u0000\u0000\u0000"+
-    "\u0296\u0297\u0005_\u0000\u0000\u0297R\u0001\u0000\u0000\u0000\u0298\u029c"+
-    "\u0003C\u001a\u0000\u0299\u029c\u0003A\u0019\u0000\u029a\u029c\u0003Q"+
-    "!\u0000\u029b\u0298\u0001\u0000\u0000\u0000\u029b\u0299\u0001\u0000\u0000"+
-    "\u0000\u029b\u029a\u0001\u0000\u0000\u0000\u029cT\u0001\u0000\u0000\u0000"+
-    "\u029d\u02a2\u0005\"\u0000\u0000\u029e\u02a1\u0003E\u001b\u0000\u029f"+
-    "\u02a1\u0003G\u001c\u0000\u02a0\u029e\u0001\u0000\u0000\u0000\u02a0\u029f"+
-    "\u0001\u0000\u0000\u0000\u02a1\u02a4\u0001\u0000\u0000\u0000\u02a2\u02a0"+
-    "\u0001\u0000\u0000\u0000\u02a2\u02a3\u0001\u0000\u0000\u0000\u02a3\u02a5"+
-    "\u0001\u0000\u0000\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5\u02bb"+
-    "\u0005\"\u0000\u0000\u02a6\u02a7\u0005\"\u0000\u0000\u02a7\u02a8\u0005"+
-    "\"\u0000\u0000\u02a8\u02a9\u0005\"\u0000\u0000\u02a9\u02ad\u0001\u0000"+
-    "\u0000\u0000\u02aa\u02ac\b\u0016\u0000\u0000\u02ab\u02aa\u0001\u0000\u0000"+
-    "\u0000\u02ac\u02af\u0001\u0000\u0000\u0000\u02ad\u02ae\u0001\u0000\u0000"+
-    "\u0000\u02ad\u02ab\u0001\u0000\u0000\u0000\u02ae\u02b0\u0001\u0000\u0000"+
-    "\u0000\u02af\u02ad\u0001\u0000\u0000\u0000\u02b0\u02b1\u0005\"\u0000\u0000"+
-    "\u02b1\u02b2\u0005\"\u0000\u0000\u02b2\u02b3\u0005\"\u0000\u0000\u02b3"+
-    "\u02b5\u0001\u0000\u0000\u0000\u02b4\u02b6\u0005\"\u0000\u0000\u02b5\u02b4"+
-    "\u0001\u0000\u0000\u0000\u02b5\u02b6\u0001\u0000\u0000\u0000\u02b6\u02b8"+
-    "\u0001\u0000\u0000\u0000\u02b7\u02b9\u0005\"\u0000\u0000\u02b8\u02b7\u0001"+
-    "\u0000\u0000\u0000\u02b8\u02b9\u0001\u0000\u0000\u0000\u02b9\u02bb\u0001"+
-    "\u0000\u0000\u0000\u02ba\u029d\u0001\u0000\u0000\u0000\u02ba\u02a6\u0001"+
-    "\u0000\u0000\u0000\u02bbV\u0001\u0000\u0000\u0000\u02bc\u02be\u0003A\u0019"+
-    "\u0000\u02bd\u02bc\u0001\u0000\u0000\u0000\u02be\u02bf\u0001\u0000\u0000"+
-    "\u0000\u02bf\u02bd\u0001\u0000\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000"+
-    "\u0000\u02c0X\u0001\u0000\u0000\u0000\u02c1\u02c3\u0003A\u0019\u0000\u02c2"+
-    "\u02c1\u0001\u0000\u0000\u0000\u02c3\u02c4\u0001\u0000\u0000\u0000\u02c4"+
-    "\u02c2\u0001\u0000\u0000\u0000\u02c4\u02c5\u0001\u0000\u0000\u0000\u02c5"+
-    "\u02c6\u0001\u0000\u0000\u0000\u02c6\u02ca\u0003i-\u0000\u02c7\u02c9\u0003"+
-    "A\u0019\u0000\u02c8\u02c7\u0001\u0000\u0000\u0000\u02c9\u02cc\u0001\u0000"+
-    "\u0000\u0000\u02ca\u02c8\u0001\u0000\u0000\u0000\u02ca\u02cb\u0001\u0000"+
-    "\u0000\u0000\u02cb\u02ec\u0001\u0000\u0000\u0000\u02cc\u02ca\u0001\u0000"+
-    "\u0000\u0000\u02cd\u02cf\u0003i-\u0000\u02ce\u02d0\u0003A\u0019\u0000"+
-    "\u02cf\u02ce\u0001\u0000\u0000\u0000\u02d0\u02d1\u0001\u0000\u0000\u0000"+
-    "\u02d1\u02cf\u0001\u0000\u0000\u0000\u02d1\u02d2\u0001\u0000\u0000\u0000"+
-    "\u02d2\u02ec\u0001\u0000\u0000\u0000\u02d3\u02d5\u0003A\u0019\u0000\u02d4"+
-    "\u02d3\u0001\u0000\u0000\u0000\u02d5\u02d6\u0001\u0000\u0000\u0000\u02d6"+
-    "\u02d4\u0001\u0000\u0000\u0000\u02d6\u02d7\u0001\u0000\u0000\u0000\u02d7"+
-    "\u02df\u0001\u0000\u0000\u0000\u02d8\u02dc\u0003i-\u0000\u02d9\u02db\u0003"+
-    "A\u0019\u0000\u02da\u02d9\u0001\u0000\u0000\u0000\u02db\u02de\u0001\u0000"+
-    "\u0000\u0000\u02dc\u02da\u0001\u0000\u0000\u0000\u02dc\u02dd\u0001\u0000"+
-    "\u0000\u0000\u02dd\u02e0\u0001\u0000\u0000\u0000\u02de\u02dc\u0001\u0000"+
-    "\u0000\u0000\u02df\u02d8\u0001\u0000\u0000\u0000\u02df\u02e0\u0001\u0000"+
-    "\u0000\u0000\u02e0\u02e1\u0001\u0000\u0000\u0000\u02e1\u02e2\u0003I\u001d"+
-    "\u0000\u02e2\u02ec\u0001\u0000\u0000\u0000\u02e3\u02e5\u0003i-\u0000\u02e4"+
-    "\u02e6\u0003A\u0019\u0000\u02e5\u02e4\u0001\u0000\u0000\u0000\u02e6\u02e7"+
-    "\u0001\u0000\u0000\u0000\u02e7\u02e5\u0001\u0000\u0000\u0000\u02e7\u02e8"+
-    "\u0001\u0000\u0000\u0000\u02e8\u02e9\u0001\u0000\u0000\u0000\u02e9\u02ea"+
-    "\u0003I\u001d\u0000\u02ea\u02ec\u0001\u0000\u0000\u0000\u02eb\u02c2\u0001"+
-    "\u0000\u0000\u0000\u02eb\u02cd\u0001\u0000\u0000\u0000\u02eb\u02d4\u0001"+
-    "\u0000\u0000\u0000\u02eb\u02e3\u0001\u0000\u0000\u0000\u02ecZ\u0001\u0000"+
-    "\u0000\u0000\u02ed\u02ee\u0007\u001e\u0000\u0000\u02ee\u02ef\u0007\u001f"+
-    "\u0000\u0000\u02ef\\\u0001\u0000\u0000\u0000\u02f0\u02f1\u0007\f\u0000"+
-    "\u0000\u02f1\u02f2\u0007\t\u0000\u0000\u02f2\u02f3\u0007\u0000\u0000\u0000"+
-    "\u02f3^\u0001\u0000\u0000\u0000\u02f4\u02f5\u0007\f\u0000\u0000\u02f5"+
-    "\u02f6\u0007\u0002\u0000\u0000\u02f6\u02f7\u0007\u0004\u0000\u0000\u02f7"+
-    "`\u0001\u0000\u0000\u0000\u02f8\u02f9\u0005=\u0000\u0000\u02f9b\u0001"+
-    "\u0000\u0000\u0000\u02fa\u02fb\u0005:\u0000\u0000\u02fb\u02fc\u0005:\u0000"+
-    "\u0000\u02fcd\u0001\u0000\u0000\u0000\u02fd\u02fe\u0005,\u0000\u0000\u02fe"+
-    "f\u0001\u0000\u0000\u0000\u02ff\u0300\u0007\u0000\u0000\u0000\u0300\u0301"+
-    "\u0007\u0003\u0000\u0000\u0301\u0302\u0007\u0002\u0000\u0000\u0302\u0303"+
-    "\u0007\u0004\u0000\u0000\u0303h\u0001\u0000\u0000\u0000\u0304\u0305\u0005"+
-    ".\u0000\u0000\u0305j\u0001\u0000\u0000\u0000\u0306\u0307\u0007\u000f\u0000"+
-    "\u0000\u0307\u0308\u0007\f\u0000\u0000\u0308\u0309\u0007\r\u0000\u0000"+
-    "\u0309\u030a\u0007\u0002\u0000\u0000\u030a\u030b\u0007\u0003\u0000\u0000"+
-    "\u030bl\u0001\u0000\u0000\u0000\u030c\u030d\u0007\u000f\u0000\u0000\u030d"+
-    "\u030e\u0007\u0001\u0000\u0000\u030e\u030f\u0007\u0006\u0000\u0000\u030f"+
-    "\u0310\u0007\u0002\u0000\u0000\u0310\u0311\u0007\u0005\u0000\u0000\u0311"+
-    "n\u0001\u0000\u0000\u0000\u0312\u0313\u0007\u0001\u0000\u0000\u0313\u0314"+
-    "\u0007\t\u0000\u0000\u0314p\u0001\u0000\u0000\u0000\u0315\u0316\u0007"+
-    "\u0001\u0000\u0000\u0316\u0317\u0007\u0002\u0000\u0000\u0317r\u0001\u0000"+
-    "\u0000\u0000\u0318\u0319\u0007\r\u0000\u0000\u0319\u031a\u0007\f\u0000"+
-    "\u0000\u031a\u031b\u0007\u0002\u0000\u0000\u031b\u031c\u0007\u0005\u0000"+
-    "\u0000\u031ct\u0001\u0000\u0000\u0000\u031d\u031e\u0007\r\u0000\u0000"+
-    "\u031e\u031f\u0007\u0001\u0000\u0000\u031f\u0320\u0007\u0012\u0000\u0000"+
-    "\u0320\u0321\u0007\u0003\u0000\u0000\u0321v\u0001\u0000\u0000\u0000\u0322"+
-    "\u0323\u0005(\u0000\u0000\u0323x\u0001\u0000\u0000\u0000\u0324\u0325\u0007"+
-    "\t\u0000\u0000\u0325\u0326\u0007\u0007\u0000\u0000\u0326\u0327\u0007\u0005"+
-    "\u0000\u0000\u0327z\u0001\u0000\u0000\u0000\u0328\u0329\u0007\t\u0000"+
-    "\u0000\u0329\u032a\u0007\u0014\u0000\u0000\u032a\u032b\u0007\r\u0000\u0000"+
-    "\u032b\u032c\u0007\r\u0000\u0000\u032c|\u0001\u0000\u0000\u0000\u032d"+
-    "\u032e\u0007\t\u0000\u0000\u032e\u032f\u0007\u0014\u0000\u0000\u032f\u0330"+
-    "\u0007\r\u0000\u0000\u0330\u0331\u0007\r\u0000\u0000\u0331\u0332\u0007"+
-    "\u0002\u0000\u0000\u0332~\u0001\u0000\u0000\u0000\u0333\u0334\u0007\u0007"+
-    "\u0000\u0000\u0334\u0335\u0007\u0006\u0000\u0000\u0335\u0080\u0001\u0000"+
-    "\u0000\u0000\u0336\u0337\u0005?\u0000\u0000\u0337\u0082\u0001\u0000\u0000"+
-    "\u0000\u0338\u0339\u0007\u0006\u0000\u0000\u0339\u033a\u0007\r\u0000\u0000"+
-    "\u033a\u033b\u0007\u0001\u0000\u0000\u033b\u033c\u0007\u0012\u0000\u0000"+
-    "\u033c\u033d\u0007\u0003\u0000\u0000\u033d\u0084\u0001\u0000\u0000\u0000"+
-    "\u033e\u033f\u0005)\u0000\u0000\u033f\u0086\u0001\u0000\u0000\u0000\u0340"+
-    "\u0341\u0007\u0005\u0000\u0000\u0341\u0342\u0007\u0006\u0000\u0000\u0342"+
-    "\u0343\u0007\u0014\u0000\u0000\u0343\u0344\u0007\u0003\u0000\u0000\u0344"+
-    "\u0088\u0001\u0000\u0000\u0000\u0345\u0346\u0005=\u0000\u0000\u0346\u0347"+
-    "\u0005=\u0000\u0000\u0347\u008a\u0001\u0000\u0000\u0000\u0348\u0349\u0005"+
-    "=\u0000\u0000\u0349\u034a\u0005~\u0000\u0000\u034a\u008c\u0001\u0000\u0000"+
-    "\u0000\u034b\u034c\u0005!\u0000\u0000\u034c\u034d\u0005=\u0000\u0000\u034d"+
-    "\u008e\u0001\u0000\u0000\u0000\u034e\u034f\u0005<\u0000\u0000\u034f\u0090"+
-    "\u0001\u0000\u0000\u0000\u0350\u0351\u0005<\u0000\u0000\u0351\u0352\u0005"+
-    "=\u0000\u0000\u0352\u0092\u0001\u0000\u0000\u0000\u0353\u0354\u0005>\u0000"+
-    "\u0000\u0354\u0094\u0001\u0000\u0000\u0000\u0355\u0356\u0005>\u0000\u0000"+
-    "\u0356\u0357\u0005=\u0000\u0000\u0357\u0096\u0001\u0000\u0000\u0000\u0358"+
-    "\u0359\u0005+\u0000\u0000\u0359\u0098\u0001\u0000\u0000\u0000\u035a\u035b"+
-    "\u0005-\u0000\u0000\u035b\u009a\u0001\u0000\u0000\u0000\u035c\u035d\u0005"+
-    "*\u0000\u0000\u035d\u009c\u0001\u0000\u0000\u0000\u035e\u035f\u0005/\u0000"+
-    "\u0000\u035f\u009e\u0001\u0000\u0000\u0000\u0360\u0361\u0005%\u0000\u0000"+
-    "\u0361\u00a0\u0001\u0000\u0000\u0000\u0362\u0363\u0004I\u0004\u0000\u0363"+
-    "\u0364\u00033\u0012\u0000\u0364\u0365\u0001\u0000\u0000\u0000\u0365\u0366"+
-    "\u0006I\f\u0000\u0366\u00a2\u0001\u0000\u0000\u0000\u0367\u036a\u0003"+
-    "\u00819\u0000\u0368\u036b\u0003C\u001a\u0000\u0369\u036b\u0003Q!\u0000"+
-    "\u036a\u0368\u0001\u0000\u0000\u0000\u036a\u0369\u0001\u0000\u0000\u0000"+
-    "\u036b\u036f\u0001\u0000\u0000\u0000\u036c\u036e\u0003S\"\u0000\u036d"+
-    "\u036c\u0001\u0000\u0000\u0000\u036e\u0371\u0001\u0000\u0000\u0000\u036f"+
-    "\u036d\u0001\u0000\u0000\u0000\u036f\u0370\u0001\u0000\u0000\u0000\u0370"+
-    "\u0379\u0001\u0000\u0000\u0000\u0371\u036f\u0001\u0000\u0000\u0000\u0372"+
-    "\u0374\u0003\u00819\u0000\u0373\u0375\u0003A\u0019\u0000\u0374\u0373\u0001"+
-    "\u0000\u0000\u0000\u0375\u0376\u0001\u0000\u0000\u0000\u0376\u0374\u0001"+
-    "\u0000\u0000\u0000\u0376\u0377\u0001\u0000\u0000\u0000\u0377\u0379\u0001"+
-    "\u0000\u0000\u0000\u0378\u0367\u0001\u0000\u0000\u0000\u0378\u0372\u0001"+
-    "\u0000\u0000\u0000\u0379\u00a4\u0001\u0000\u0000\u0000\u037a\u037b\u0005"+
-    "[\u0000\u0000\u037b\u037c\u0001\u0000\u0000\u0000\u037c\u037d\u0006K\u0000"+
-    "\u0000\u037d\u037e\u0006K\u0000\u0000\u037e\u00a6\u0001\u0000\u0000\u0000"+
-    "\u037f\u0380\u0005]\u0000\u0000\u0380\u0381\u0001\u0000\u0000\u0000\u0381"+
-    "\u0382\u0006L\u000b\u0000\u0382\u0383\u0006L\u000b\u0000\u0383\u00a8\u0001"+
-    "\u0000\u0000\u0000\u0384\u0388\u0003C\u001a\u0000\u0385\u0387\u0003S\""+
-    "\u0000\u0386\u0385\u0001\u0000\u0000\u0000\u0387\u038a\u0001\u0000\u0000"+
-    "\u0000\u0388\u0386\u0001\u0000\u0000\u0000\u0388\u0389\u0001\u0000\u0000"+
-    "\u0000\u0389\u0395\u0001\u0000\u0000\u0000\u038a\u0388\u0001\u0000\u0000"+
-    "\u0000\u038b\u038e\u0003Q!\u0000\u038c\u038e\u0003K\u001e\u0000\u038d"+
-    "\u038b\u0001\u0000\u0000\u0000\u038d\u038c\u0001\u0000\u0000\u0000\u038e"+
-    "\u0390\u0001\u0000\u0000\u0000\u038f\u0391\u0003S\"\u0000\u0390\u038f"+
-    "\u0001\u0000\u0000\u0000\u0391\u0392\u0001\u0000\u0000\u0000\u0392\u0390"+
-    "\u0001\u0000\u0000\u0000\u0392\u0393\u0001\u0000\u0000\u0000\u0393\u0395"+
-    "\u0001\u0000\u0000\u0000\u0394\u0384\u0001\u0000\u0000\u0000\u0394\u038d"+
-    "\u0001\u0000\u0000\u0000\u0395\u00aa\u0001\u0000\u0000\u0000\u0396\u0398"+
-    "\u0003M\u001f\u0000\u0397\u0399\u0003O \u0000\u0398\u0397\u0001\u0000"+
-    "\u0000\u0000\u0399\u039a\u0001\u0000\u0000\u0000\u039a\u0398\u0001\u0000"+
-    "\u0000\u0000\u039a\u039b\u0001\u0000\u0000\u0000\u039b\u039c\u0001\u0000"+
-    "\u0000\u0000\u039c\u039d\u0003M\u001f\u0000\u039d\u00ac\u0001\u0000\u0000"+
-    "\u0000\u039e\u039f\u0003\u00abN\u0000\u039f\u00ae\u0001\u0000\u0000\u0000"+
-    "\u03a0\u03a1\u00039\u0015\u0000\u03a1\u03a2\u0001\u0000\u0000\u0000\u03a2"+
-    "\u03a3\u0006P\n\u0000\u03a3\u00b0\u0001\u0000\u0000\u0000\u03a4\u03a5"+
-    "\u0003;\u0016\u0000\u03a5\u03a6\u0001\u0000\u0000\u0000\u03a6\u03a7\u0006"+
-    "Q\n\u0000\u03a7\u00b2\u0001\u0000\u0000\u0000\u03a8\u03a9\u0003=\u0017"+
-    "\u0000\u03a9\u03aa\u0001\u0000\u0000\u0000\u03aa\u03ab\u0006R\n\u0000"+
-    "\u03ab\u00b4\u0001\u0000\u0000\u0000\u03ac\u03ad\u0003\u00a5K\u0000\u03ad"+
-    "\u03ae\u0001\u0000\u0000\u0000\u03ae\u03af\u0006S\r\u0000\u03af\u03b0"+
-    "\u0006S\u000e\u0000\u03b0\u00b6\u0001\u0000\u0000\u0000\u03b1\u03b2\u0003"+
-    "?\u0018\u0000\u03b2\u03b3\u0001\u0000\u0000\u0000\u03b3\u03b4\u0006T\u000f"+
-    "\u0000\u03b4\u03b5\u0006T\u000b\u0000\u03b5\u00b8\u0001\u0000\u0000\u0000"+
-    "\u03b6\u03b7\u0003=\u0017\u0000\u03b7\u03b8\u0001\u0000\u0000\u0000\u03b8"+
-    "\u03b9\u0006U\n\u0000\u03b9\u00ba\u0001\u0000\u0000\u0000\u03ba\u03bb"+
-    "\u00039\u0015\u0000\u03bb\u03bc\u0001\u0000\u0000\u0000\u03bc\u03bd\u0006"+
-    "V\n\u0000\u03bd\u00bc\u0001\u0000\u0000\u0000\u03be\u03bf\u0003;\u0016"+
-    "\u0000\u03bf\u03c0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006W\n\u0000"+
-    "\u03c1\u00be\u0001\u0000\u0000\u0000\u03c2\u03c3\u0003?\u0018\u0000\u03c3"+
-    "\u03c4\u0001\u0000\u0000\u0000\u03c4\u03c5\u0006X\u000f\u0000\u03c5\u03c6"+
-    "\u0006X\u000b\u0000\u03c6\u00c0\u0001\u0000\u0000\u0000\u03c7\u03c8\u0003"+
-    "\u00a5K\u0000\u03c8\u03c9\u0001\u0000\u0000\u0000\u03c9\u03ca\u0006Y\r"+
-    "\u0000\u03ca\u00c2\u0001\u0000\u0000\u0000\u03cb\u03cc\u0003\u00a7L\u0000"+
-    "\u03cc\u03cd\u0001\u0000\u0000\u0000\u03cd\u03ce\u0006Z\u0010\u0000\u03ce"+
-    "\u00c4\u0001\u0000\u0000\u0000\u03cf\u03d0\u0003\u0151\u00a1\u0000\u03d0"+
-    "\u03d1\u0001\u0000\u0000\u0000\u03d1\u03d2\u0006[\u0011\u0000\u03d2\u00c6"+
-    "\u0001\u0000\u0000\u0000\u03d3\u03d4\u0003e+\u0000\u03d4\u03d5\u0001\u0000"+
-    "\u0000\u0000\u03d5\u03d6\u0006\\\u0012\u0000\u03d6\u00c8\u0001\u0000\u0000"+
-    "\u0000\u03d7\u03d8\u0003a)\u0000\u03d8\u03d9\u0001\u0000\u0000\u0000\u03d9"+
-    "\u03da\u0006]\u0013\u0000\u03da\u00ca\u0001\u0000\u0000\u0000\u03db\u03dc"+
-    "\u0007\u0010\u0000\u0000\u03dc\u03dd\u0007\u0003\u0000\u0000\u03dd\u03de"+
-    "\u0007\u0005\u0000\u0000\u03de\u03df\u0007\f\u0000\u0000\u03df\u03e0\u0007"+
-    "\u0000\u0000\u0000\u03e0\u03e1\u0007\f\u0000\u0000\u03e1\u03e2\u0007\u0005"+
-    "\u0000\u0000\u03e2\u03e3\u0007\f\u0000\u0000\u03e3\u00cc\u0001\u0000\u0000"+
-    "\u0000\u03e4\u03e8\b \u0000\u0000\u03e5\u03e6\u0005/\u0000\u0000\u03e6"+
-    "\u03e8\b!\u0000\u0000\u03e7\u03e4\u0001\u0000\u0000\u0000\u03e7\u03e5"+
-    "\u0001\u0000\u0000\u0000\u03e8\u00ce\u0001\u0000\u0000\u0000\u03e9\u03eb"+
-    "\u0003\u00cd_\u0000\u03ea\u03e9\u0001\u0000\u0000\u0000\u03eb\u03ec\u0001"+
-    "\u0000\u0000\u0000\u03ec\u03ea\u0001\u0000\u0000\u0000\u03ec\u03ed\u0001"+
-    "\u0000\u0000\u0000\u03ed\u00d0\u0001\u0000\u0000\u0000\u03ee\u03ef\u0003"+
-    "\u00cf`\u0000\u03ef\u03f0\u0001\u0000\u0000\u0000\u03f0\u03f1\u0006a\u0014"+
-    "\u0000\u03f1\u00d2\u0001\u0000\u0000\u0000\u03f2\u03f3\u0003U#\u0000\u03f3"+
-    "\u03f4\u0001\u0000\u0000\u0000\u03f4\u03f5\u0006b\u0015\u0000\u03f5\u00d4"+
-    "\u0001\u0000\u0000\u0000\u03f6\u03f7\u00039\u0015\u0000\u03f7\u03f8\u0001"+
-    "\u0000\u0000\u0000\u03f8\u03f9\u0006c\n\u0000\u03f9\u00d6\u0001\u0000"+
-    "\u0000\u0000\u03fa\u03fb\u0003;\u0016\u0000\u03fb\u03fc\u0001\u0000\u0000"+
-    "\u0000\u03fc\u03fd\u0006d\n\u0000\u03fd\u00d8\u0001\u0000\u0000\u0000"+
-    "\u03fe\u03ff\u0003=\u0017\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400"+
-    "\u0401\u0006e\n\u0000\u0401\u00da\u0001\u0000\u0000\u0000\u0402\u0403"+
-    "\u0003?\u0018\u0000\u0403\u0404\u0001\u0000\u0000\u0000\u0404\u0405\u0006"+
-    "f\u000f\u0000\u0405\u0406\u0006f\u000b\u0000\u0406\u00dc\u0001\u0000\u0000"+
-    "\u0000\u0407\u0408\u0003i-\u0000\u0408\u0409\u0001\u0000\u0000\u0000\u0409"+
-    "\u040a\u0006g\u0016\u0000\u040a\u00de\u0001\u0000\u0000\u0000\u040b\u040c"+
-    "\u0003e+\u0000\u040c\u040d\u0001\u0000\u0000\u0000\u040d\u040e\u0006h"+
-    "\u0012\u0000\u040e\u00e0\u0001\u0000\u0000\u0000\u040f\u0410\u0003\u0081"+
-    "9\u0000\u0410\u0411\u0001\u0000\u0000\u0000\u0411\u0412\u0006i\u0017\u0000"+
-    "\u0412\u00e2\u0001\u0000\u0000\u0000\u0413\u0414\u0003\u00a3J\u0000\u0414"+
-    "\u0415\u0001\u0000\u0000\u0000\u0415\u0416\u0006j\u0018\u0000\u0416\u00e4"+
-    "\u0001\u0000\u0000\u0000\u0417\u041c\u0003C\u001a\u0000\u0418\u041c\u0003"+
-    "A\u0019\u0000\u0419\u041c\u0003Q!\u0000\u041a\u041c\u0003\u009bF\u0000"+
-    "\u041b\u0417\u0001\u0000\u0000\u0000\u041b\u0418\u0001\u0000\u0000\u0000"+
-    "\u041b\u0419\u0001\u0000\u0000\u0000\u041b\u041a\u0001\u0000\u0000\u0000"+
-    "\u041c\u00e6\u0001\u0000\u0000\u0000\u041d\u0420\u0003C\u001a\u0000\u041e"+
-    "\u0420\u0003\u009bF\u0000\u041f\u041d\u0001\u0000\u0000\u0000\u041f\u041e"+
-    "\u0001\u0000\u0000\u0000\u0420\u0424\u0001\u0000\u0000\u0000\u0421\u0423"+
-    "\u0003\u00e5k\u0000\u0422\u0421\u0001\u0000\u0000\u0000\u0423\u0426\u0001"+
-    "\u0000\u0000\u0000\u0424\u0422\u0001\u0000\u0000\u0000\u0424\u0425\u0001"+
-    "\u0000\u0000\u0000\u0425\u0431\u0001\u0000\u0000\u0000\u0426\u0424\u0001"+
-    "\u0000\u0000\u0000\u0427\u042a\u0003Q!\u0000\u0428\u042a\u0003K\u001e"+
-    "\u0000\u0429\u0427\u0001\u0000\u0000\u0000\u0429\u0428\u0001\u0000\u0000"+
-    "\u0000\u042a\u042c\u0001\u0000\u0000\u0000\u042b\u042d\u0003\u00e5k\u0000"+
-    "\u042c\u042b\u0001\u0000\u0000\u0000\u042d\u042e\u0001\u0000\u0000\u0000"+
-    "\u042e\u042c\u0001\u0000\u0000\u0000\u042e\u042f\u0001\u0000\u0000\u0000"+
-    "\u042f\u0431\u0001\u0000\u0000\u0000\u0430\u041f\u0001\u0000\u0000\u0000"+
-    "\u0430\u0429\u0001\u0000\u0000\u0000\u0431\u00e8\u0001\u0000\u0000\u0000"+
-    "\u0432\u0435\u0003\u00e7l\u0000\u0433\u0435\u0003\u00abN\u0000\u0434\u0432"+
-    "\u0001\u0000\u0000\u0000\u0434\u0433\u0001\u0000\u0000\u0000\u0435\u0436"+
-    "\u0001\u0000\u0000\u0000\u0436\u0434\u0001\u0000\u0000\u0000\u0436\u0437"+
-    "\u0001\u0000\u0000\u0000\u0437\u00ea\u0001\u0000\u0000\u0000\u0438\u0439"+
-    "\u00039\u0015\u0000\u0439\u043a\u0001\u0000\u0000\u0000\u043a\u043b\u0006"+
-    "n\n\u0000\u043b\u00ec\u0001\u0000\u0000\u0000\u043c\u043d\u0003;\u0016"+
-    "\u0000\u043d\u043e\u0001\u0000\u0000\u0000\u043e\u043f\u0006o\n\u0000"+
-    "\u043f\u00ee\u0001\u0000\u0000\u0000\u0440\u0441\u0003=\u0017\u0000\u0441"+
-    "\u0442\u0001\u0000\u0000\u0000\u0442\u0443\u0006p\n\u0000\u0443\u00f0"+
-    "\u0001\u0000\u0000\u0000\u0444\u0445\u0003?\u0018\u0000\u0445\u0446\u0001"+
-    "\u0000\u0000\u0000\u0446\u0447\u0006q\u000f\u0000\u0447\u0448\u0006q\u000b"+
-    "\u0000\u0448\u00f2\u0001\u0000\u0000\u0000\u0449\u044a\u0003a)\u0000\u044a"+
-    "\u044b\u0001\u0000\u0000\u0000\u044b\u044c\u0006r\u0013\u0000\u044c\u00f4"+
-    "\u0001\u0000\u0000\u0000\u044d\u044e\u0003e+\u0000\u044e\u044f\u0001\u0000"+
-    "\u0000\u0000\u044f\u0450\u0006s\u0012\u0000\u0450\u00f6\u0001\u0000\u0000"+
-    "\u0000\u0451\u0452\u0003i-\u0000\u0452\u0453\u0001\u0000\u0000\u0000\u0453"+
-    "\u0454\u0006t\u0016\u0000\u0454\u00f8\u0001\u0000\u0000\u0000\u0455\u0456"+
-    "\u0003\u00819\u0000\u0456\u0457\u0001\u0000\u0000\u0000\u0457\u0458\u0006"+
-    "u\u0017\u0000\u0458\u00fa\u0001\u0000\u0000\u0000\u0459\u045a\u0003\u00a3"+
-    "J\u0000\u045a\u045b\u0001\u0000\u0000\u0000\u045b\u045c\u0006v\u0018\u0000"+
-    "\u045c\u00fc\u0001\u0000\u0000\u0000\u045d\u045e\u0007\f\u0000\u0000\u045e"+
-    "\u045f\u0007\u0002\u0000\u0000\u045f\u00fe\u0001\u0000\u0000\u0000\u0460"+
-    "\u0461\u0003\u00e9m\u0000\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463"+
-    "\u0006x\u0019\u0000\u0463\u0100\u0001\u0000\u0000\u0000\u0464\u0465\u0003"+
-    "9\u0015\u0000\u0465\u0466\u0001\u0000\u0000\u0000\u0466\u0467\u0006y\n"+
-    "\u0000\u0467\u0102\u0001\u0000\u0000\u0000\u0468\u0469\u0003;\u0016\u0000"+
-    "\u0469\u046a\u0001\u0000\u0000\u0000\u046a\u046b\u0006z\n\u0000\u046b"+
-    "\u0104\u0001\u0000\u0000\u0000\u046c\u046d\u0003=\u0017\u0000\u046d\u046e"+
-    "\u0001\u0000\u0000\u0000\u046e\u046f\u0006{\n\u0000\u046f\u0106\u0001"+
-    "\u0000\u0000\u0000\u0470\u0471\u0003?\u0018\u0000\u0471\u0472\u0001\u0000"+
-    "\u0000\u0000\u0472\u0473\u0006|\u000f\u0000\u0473\u0474\u0006|\u000b\u0000"+
-    "\u0474\u0108\u0001\u0000\u0000\u0000\u0475\u0476\u0003\u00a5K\u0000\u0476"+
-    "\u0477\u0001\u0000\u0000\u0000\u0477\u0478\u0006}\r\u0000\u0478\u0479"+
-    "\u0006}\u001a\u0000\u0479\u010a\u0001\u0000\u0000\u0000\u047a\u047b\u0007"+
-    "\u0007\u0000\u0000\u047b\u047c\u0007\t\u0000\u0000\u047c\u047d\u0001\u0000"+
-    "\u0000\u0000\u047d\u047e\u0006~\u001b\u0000\u047e\u010c\u0001\u0000\u0000"+
-    "\u0000\u047f\u0480\u0007\u0013\u0000\u0000\u0480\u0481\u0007\u0001\u0000"+
-    "\u0000\u0481\u0482\u0007\u0005\u0000\u0000\u0482\u0483\u0007\n\u0000\u0000"+
-    "\u0483\u0484\u0001\u0000\u0000\u0000\u0484\u0485\u0006\u007f\u001b\u0000"+
-    "\u0485\u010e\u0001\u0000\u0000\u0000\u0486\u0487\b\"\u0000\u0000\u0487"+
-    "\u0110\u0001\u0000\u0000\u0000\u0488\u048a\u0003\u010f\u0080\u0000\u0489"+
-    "\u0488\u0001\u0000\u0000\u0000\u048a\u048b\u0001\u0000\u0000\u0000\u048b"+
-    "\u0489\u0001\u0000\u0000\u0000\u048b\u048c\u0001\u0000\u0000\u0000\u048c"+
-    "\u048d\u0001\u0000\u0000\u0000\u048d\u048e\u0003\u0151\u00a1\u0000\u048e"+
-    "\u0490\u0001\u0000\u0000\u0000\u048f\u0489\u0001\u0000\u0000\u0000\u048f"+
-    "\u0490\u0001\u0000\u0000\u0000\u0490\u0492\u0001\u0000\u0000\u0000\u0491"+
-    "\u0493\u0003\u010f\u0080\u0000\u0492\u0491\u0001\u0000\u0000\u0000\u0493"+
-    "\u0494\u0001\u0000\u0000\u0000\u0494\u0492\u0001\u0000\u0000\u0000\u0494"+
-    "\u0495\u0001\u0000\u0000\u0000\u0495\u0112\u0001\u0000\u0000\u0000\u0496"+
-    "\u0497\u0003\u0111\u0081\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498"+
-    "\u0499\u0006\u0082\u001c\u0000\u0499\u0114\u0001\u0000\u0000\u0000\u049a"+
-    "\u049b\u00039\u0015\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d"+
-    "\u0006\u0083\n\u0000\u049d\u0116\u0001\u0000\u0000\u0000\u049e\u049f\u0003"+
-    ";\u0016\u0000\u049f\u04a0\u0001\u0000\u0000\u0000\u04a0\u04a1\u0006\u0084"+
-    "\n\u0000\u04a1\u0118\u0001\u0000\u0000\u0000\u04a2\u04a3\u0003=\u0017"+
-    "\u0000\u04a3\u04a4\u0001\u0000\u0000\u0000\u04a4\u04a5\u0006\u0085\n\u0000"+
-    "\u04a5\u011a\u0001\u0000\u0000\u0000\u04a6\u04a7\u0003?\u0018\u0000\u04a7"+
-    "\u04a8\u0001\u0000\u0000\u0000\u04a8\u04a9\u0006\u0086\u000f\u0000\u04a9"+
-    "\u04aa\u0006\u0086\u000b\u0000\u04aa\u04ab\u0006\u0086\u000b\u0000\u04ab"+
-    "\u011c\u0001\u0000\u0000\u0000\u04ac\u04ad\u0003a)\u0000\u04ad\u04ae\u0001"+
-    "\u0000\u0000\u0000\u04ae\u04af\u0006\u0087\u0013\u0000\u04af\u011e\u0001"+
-    "\u0000\u0000\u0000\u04b0\u04b1\u0003e+\u0000\u04b1\u04b2\u0001\u0000\u0000"+
-    "\u0000\u04b2\u04b3\u0006\u0088\u0012\u0000\u04b3\u0120\u0001\u0000\u0000"+
-    "\u0000\u04b4\u04b5\u0003i-\u0000\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6"+
-    "\u04b7\u0006\u0089\u0016\u0000\u04b7\u0122\u0001\u0000\u0000\u0000\u04b8"+
-    "\u04b9\u0003\u010d\u007f\u0000\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba"+
-    "\u04bb\u0006\u008a\u001d\u0000\u04bb\u0124\u0001\u0000\u0000\u0000\u04bc"+
-    "\u04bd\u0003\u00e9m\u0000\u04bd\u04be\u0001\u0000\u0000\u0000\u04be\u04bf"+
-    "\u0006\u008b\u0019\u0000\u04bf\u0126\u0001\u0000\u0000\u0000\u04c0\u04c1"+
-    "\u0003\u00adO\u0000\u04c1\u04c2\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006"+
-    "\u008c\u001e\u0000\u04c3\u0128\u0001\u0000\u0000\u0000\u04c4\u04c5\u0003"+
-    "\u00819\u0000\u04c5\u04c6\u0001\u0000\u0000\u0000\u04c6\u04c7\u0006\u008d"+
-    "\u0017\u0000\u04c7\u012a\u0001\u0000\u0000\u0000\u04c8\u04c9\u0003\u00a3"+
-    "J\u0000\u04c9\u04ca\u0001\u0000\u0000\u0000\u04ca\u04cb\u0006\u008e\u0018"+
-    "\u0000\u04cb\u012c\u0001\u0000\u0000\u0000\u04cc\u04cd\u00039\u0015\u0000"+
-    "\u04cd\u04ce\u0001\u0000\u0000\u0000\u04ce\u04cf\u0006\u008f\n\u0000\u04cf"+
-    "\u012e\u0001\u0000\u0000\u0000\u04d0\u04d1\u0003;\u0016\u0000\u04d1\u04d2"+
-    "\u0001\u0000\u0000\u0000\u04d2\u04d3\u0006\u0090\n\u0000\u04d3\u0130\u0001"+
-    "\u0000\u0000\u0000\u04d4\u04d5\u0003=\u0017\u0000\u04d5\u04d6\u0001\u0000"+
-    "\u0000\u0000\u04d6\u04d7\u0006\u0091\n\u0000\u04d7\u0132\u0001\u0000\u0000"+
-    "\u0000\u04d8\u04d9\u0003?\u0018\u0000\u04d9\u04da\u0001\u0000\u0000\u0000"+
-    "\u04da\u04db\u0006\u0092\u000f\u0000\u04db\u04dc\u0006\u0092\u000b\u0000"+
-    "\u04dc\u0134\u0001\u0000\u0000\u0000\u04dd\u04de\u0003i-\u0000\u04de\u04df"+
-    "\u0001\u0000\u0000\u0000\u04df\u04e0\u0006\u0093\u0016\u0000\u04e0\u0136"+
-    "\u0001\u0000\u0000\u0000\u04e1\u04e2\u0003\u00819\u0000\u04e2\u04e3\u0001"+
-    "\u0000\u0000\u0000\u04e3\u04e4\u0006\u0094\u0017\u0000\u04e4\u0138\u0001"+
-    "\u0000\u0000\u0000\u04e5\u04e6\u0003\u00a3J\u0000\u04e6\u04e7\u0001\u0000"+
-    "\u0000\u0000\u04e7\u04e8\u0006\u0095\u0018\u0000\u04e8\u013a\u0001\u0000"+
-    "\u0000\u0000\u04e9\u04ea\u0003\u00adO\u0000\u04ea\u04eb\u0001\u0000\u0000"+
-    "\u0000\u04eb\u04ec\u0006\u0096\u001e\u0000\u04ec\u013c\u0001\u0000\u0000"+
-    "\u0000\u04ed\u04ee\u0003\u00a9M\u0000\u04ee\u04ef\u0001\u0000\u0000\u0000"+
-    "\u04ef\u04f0\u0006\u0097\u001f\u0000\u04f0\u013e\u0001\u0000\u0000\u0000"+
-    "\u04f1\u04f2\u00039\u0015\u0000\u04f2\u04f3\u0001\u0000\u0000\u0000\u04f3"+
-    "\u04f4\u0006\u0098\n\u0000\u04f4\u0140\u0001\u0000\u0000\u0000\u04f5\u04f6"+
-    "\u0003;\u0016\u0000\u04f6\u04f7\u0001\u0000\u0000\u0000\u04f7\u04f8\u0006"+
-    "\u0099\n\u0000\u04f8\u0142\u0001\u0000\u0000\u0000\u04f9\u04fa\u0003="+
-    "\u0017\u0000\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u009a"+
-    "\n\u0000\u04fc\u0144\u0001\u0000\u0000\u0000\u04fd\u04fe\u0003?\u0018"+
-    "\u0000\u04fe\u04ff\u0001\u0000\u0000\u0000\u04ff\u0500\u0006\u009b\u000f"+
-    "\u0000\u0500\u0501\u0006\u009b\u000b\u0000\u0501\u0146\u0001\u0000\u0000"+
-    "\u0000\u0502\u0503\u0007\u0001\u0000\u0000\u0503\u0504\u0007\t\u0000\u0000"+
-    "\u0504\u0505\u0007\u000f\u0000\u0000\u0505\u0506\u0007\u0007\u0000\u0000"+
-    "\u0506\u0148\u0001\u0000\u0000\u0000\u0507\u0508\u00039\u0015\u0000\u0508"+
-    "\u0509\u0001\u0000\u0000\u0000\u0509\u050a\u0006\u009d\n\u0000\u050a\u014a"+
-    "\u0001\u0000\u0000\u0000\u050b\u050c\u0003;\u0016\u0000\u050c\u050d\u0001"+
-    "\u0000\u0000\u0000\u050d\u050e\u0006\u009e\n\u0000\u050e\u014c\u0001\u0000"+
-    "\u0000\u0000\u050f\u0510\u0003=\u0017\u0000\u0510\u0511\u0001\u0000\u0000"+
-    "\u0000\u0511\u0512\u0006\u009f\n\u0000\u0512\u014e\u0001\u0000\u0000\u0000"+
-    "\u0513\u0514\u0003\u00a7L\u0000\u0514\u0515\u0001\u0000\u0000\u0000\u0515"+
-    "\u0516\u0006\u00a0\u0010\u0000\u0516\u0517\u0006\u00a0\u000b\u0000\u0517"+
-    "\u0150\u0001\u0000\u0000\u0000\u0518\u0519\u0005:\u0000\u0000\u0519\u0152"+
-    "\u0001\u0000\u0000\u0000\u051a\u0520\u0003K\u001e\u0000\u051b\u0520\u0003"+
-    "A\u0019\u0000\u051c\u0520\u0003i-\u0000\u051d\u0520\u0003C\u001a\u0000"+
-    "\u051e\u0520\u0003Q!\u0000\u051f\u051a\u0001\u0000\u0000\u0000\u051f\u051b"+
-    "\u0001\u0000\u0000\u0000\u051f\u051c\u0001\u0000\u0000\u0000\u051f\u051d"+
-    "\u0001\u0000\u0000\u0000\u051f\u051e\u0001\u0000\u0000\u0000\u0520\u0521"+
-    "\u0001\u0000\u0000\u0000\u0521\u051f\u0001\u0000\u0000\u0000\u0521\u0522"+
-    "\u0001\u0000\u0000\u0000\u0522\u0154\u0001\u0000\u0000\u0000\u0523\u0524"+
-    "\u00039\u0015\u0000\u0524\u0525\u0001\u0000\u0000\u0000\u0525\u0526\u0006"+
-    "\u00a3\n\u0000\u0526\u0156\u0001\u0000\u0000\u0000\u0527\u0528\u0003;"+
-    "\u0016\u0000\u0528\u0529\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u00a4"+
-    "\n\u0000\u052a\u0158\u0001\u0000\u0000\u0000\u052b\u052c\u0003=\u0017"+
-    "\u0000\u052c\u052d\u0001\u0000\u0000\u0000\u052d\u052e\u0006\u00a5\n\u0000"+
-    "\u052e\u015a\u0001\u0000\u0000\u0000\u052f\u0530\u0003?\u0018\u0000\u0530"+
-    "\u0531\u0001\u0000\u0000\u0000\u0531\u0532\u0006\u00a6\u000f\u0000\u0532"+
-    "\u0533\u0006\u00a6\u000b\u0000\u0533\u015c\u0001\u0000\u0000\u0000\u0534"+
-    "\u0535\u0003\u0151\u00a1\u0000\u0535\u0536\u0001\u0000\u0000\u0000\u0536"+
-    "\u0537\u0006\u00a7\u0011\u0000\u0537\u015e\u0001\u0000\u0000\u0000\u0538"+
-    "\u0539\u0003e+\u0000\u0539\u053a\u0001\u0000\u0000\u0000\u053a\u053b\u0006"+
-    "\u00a8\u0012\u0000\u053b\u0160\u0001\u0000\u0000\u0000\u053c\u053d\u0003"+
-    "i-\u0000\u053d\u053e\u0001\u0000\u0000\u0000\u053e\u053f\u0006\u00a9\u0016"+
-    "\u0000\u053f\u0162\u0001\u0000\u0000\u0000\u0540\u0541\u0003\u010b~\u0000"+
-    "\u0541\u0542\u0001\u0000\u0000\u0000\u0542\u0543\u0006\u00aa \u0000\u0543"+
-    "\u0544\u0006\u00aa!\u0000\u0544\u0164\u0001\u0000\u0000\u0000\u0545\u0546"+
-    "\u0003\u00cf`\u0000\u0546\u0547\u0001\u0000\u0000\u0000\u0547\u0548\u0006"+
-    "\u00ab\u0014\u0000\u0548\u0166\u0001\u0000\u0000\u0000\u0549\u054a\u0003"+
-    "U#\u0000\u054a\u054b\u0001\u0000\u0000\u0000\u054b\u054c\u0006\u00ac\u0015"+
-    "\u0000\u054c\u0168\u0001\u0000\u0000\u0000\u054d\u054e\u00039\u0015\u0000"+
-    "\u054e\u054f\u0001\u0000\u0000\u0000\u054f\u0550\u0006\u00ad\n\u0000\u0550"+
-    "\u016a\u0001\u0000\u0000\u0000\u0551\u0552\u0003;\u0016\u0000\u0552\u0553"+
-    "\u0001\u0000\u0000\u0000\u0553\u0554\u0006\u00ae\n\u0000\u0554\u016c\u0001"+
-    "\u0000\u0000\u0000\u0555\u0556\u0003=\u0017\u0000\u0556\u0557\u0001\u0000"+
-    "\u0000\u0000\u0557\u0558\u0006\u00af\n\u0000\u0558\u016e\u0001\u0000\u0000"+
-    "\u0000\u0559\u055a\u0003?\u0018\u0000\u055a\u055b\u0001\u0000\u0000\u0000"+
-    "\u055b\u055c\u0006\u00b0\u000f\u0000\u055c\u055d\u0006\u00b0\u000b\u0000"+
-    "\u055d\u055e\u0006\u00b0\u000b\u0000\u055e\u0170\u0001\u0000\u0000\u0000"+
-    "\u055f\u0560\u0003e+\u0000\u0560\u0561\u0001\u0000\u0000\u0000\u0561\u0562"+
-    "\u0006\u00b1\u0012\u0000\u0562\u0172\u0001\u0000\u0000\u0000\u0563\u0564"+
-    "\u0003i-\u0000\u0564\u0565\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00b2"+
-    "\u0016\u0000\u0566\u0174\u0001\u0000\u0000\u0000\u0567\u0568\u0003\u00e9"+
-    "m\u0000\u0568\u0569\u0001\u0000\u0000\u0000\u0569\u056a\u0006\u00b3\u0019"+
-    "\u0000\u056a\u0176\u0001\u0000\u0000\u0000\u056b\u056c\u00039\u0015\u0000"+
-    "\u056c\u056d\u0001\u0000\u0000\u0000\u056d\u056e\u0006\u00b4\n\u0000\u056e"+
-    "\u0178\u0001\u0000\u0000\u0000\u056f\u0570\u0003;\u0016\u0000\u0570\u0571"+
-    "\u0001\u0000\u0000\u0000\u0571\u0572\u0006\u00b5\n\u0000\u0572\u017a\u0001"+
-    "\u0000\u0000\u0000\u0573\u0574\u0003=\u0017\u0000\u0574\u0575\u0001\u0000"+
-    "\u0000\u0000\u0575\u0576\u0006\u00b6\n\u0000\u0576\u017c\u0001\u0000\u0000"+
-    "\u0000\u0577\u0578\u0003?\u0018\u0000\u0578\u0579\u0001\u0000\u0000\u0000"+
-    "\u0579\u057a\u0006\u00b7\u000f\u0000\u057a\u057b\u0006\u00b7\u000b\u0000"+
-    "\u057b\u017e\u0001\u0000\u0000\u0000\u057c\u057d\u0003\u00cf`\u0000\u057d"+
-    "\u057e\u0001\u0000\u0000\u0000\u057e\u057f\u0006\u00b8\u0014\u0000\u057f"+
-    "\u0580\u0006\u00b8\u000b\u0000\u0580\u0581\u0006\u00b8\"\u0000\u0581\u0180"+
-    "\u0001\u0000\u0000\u0000\u0582\u0583\u0003U#\u0000\u0583\u0584\u0001\u0000"+
-    "\u0000\u0000\u0584\u0585\u0006\u00b9\u0015\u0000\u0585\u0586\u0006\u00b9"+
-    "\u000b\u0000\u0586\u0587\u0006\u00b9\"\u0000\u0587\u0182\u0001\u0000\u0000"+
-    "\u0000\u0588\u0589\u00039\u0015\u0000\u0589\u058a\u0001\u0000\u0000\u0000"+
-    "\u058a\u058b\u0006\u00ba\n\u0000\u058b\u0184\u0001\u0000\u0000\u0000\u058c"+
-    "\u058d\u0003;\u0016\u0000\u058d\u058e\u0001\u0000\u0000\u0000\u058e\u058f"+
-    "\u0006\u00bb\n\u0000\u058f\u0186\u0001\u0000\u0000\u0000\u0590\u0591\u0003"+
-    "=\u0017\u0000\u0591\u0592\u0001\u0000\u0000\u0000\u0592\u0593\u0006\u00bc"+
-    "\n\u0000\u0593\u0188\u0001\u0000\u0000\u0000\u0594\u0595\u0003\u0151\u00a1"+
-    "\u0000\u0595\u0596\u0001\u0000\u0000\u0000\u0596\u0597\u0006\u00bd\u0011"+
-    "\u0000\u0597\u0598\u0006\u00bd\u000b\u0000\u0598\u0599\u0006\u00bd\t\u0000"+
-    "\u0599\u018a\u0001\u0000\u0000\u0000\u059a\u059b\u0003e+\u0000\u059b\u059c"+
-    "\u0001\u0000\u0000\u0000\u059c\u059d\u0006\u00be\u0012\u0000\u059d\u059e"+
-    "\u0006\u00be\u000b\u0000\u059e\u059f\u0006\u00be\t\u0000\u059f\u018c\u0001"+
-    "\u0000\u0000\u0000\u05a0\u05a1\u00039\u0015\u0000\u05a1\u05a2\u0001\u0000"+
-    "\u0000\u0000\u05a2\u05a3\u0006\u00bf\n\u0000\u05a3\u018e\u0001\u0000\u0000"+
-    "\u0000\u05a4\u05a5\u0003;\u0016\u0000\u05a5\u05a6\u0001\u0000\u0000\u0000"+
-    "\u05a6\u05a7\u0006\u00c0\n\u0000\u05a7\u0190\u0001\u0000\u0000\u0000\u05a8"+
-    "\u05a9\u0003=\u0017\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa\u05ab"+
-    "\u0006\u00c1\n\u0000\u05ab\u0192\u0001\u0000\u0000\u0000\u05ac\u05ad\u0003"+
-    "\u00adO\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006\u00c2"+
-    "\u000b\u0000\u05af\u05b0\u0006\u00c2\u0000\u0000\u05b0\u05b1\u0006\u00c2"+
-    "\u001e\u0000\u05b1\u0194\u0001\u0000\u0000\u0000\u05b2\u05b3\u0003\u00a9"+
-    "M\u0000\u05b3\u05b4\u0001\u0000\u0000\u0000\u05b4\u05b5\u0006\u00c3\u000b"+
-    "\u0000\u05b5\u05b6\u0006\u00c3\u0000\u0000\u05b6\u05b7\u0006\u00c3\u001f"+
-    "\u0000\u05b7\u0196\u0001\u0000\u0000\u0000\u05b8\u05b9\u0003[&\u0000\u05b9"+
-    "\u05ba\u0001\u0000\u0000\u0000\u05ba\u05bb\u0006\u00c4\u000b\u0000\u05bb"+
-    "\u05bc\u0006\u00c4\u0000\u0000\u05bc\u05bd\u0006\u00c4#\u0000\u05bd\u0198"+
-    "\u0001\u0000\u0000\u0000\u05be\u05bf\u0003?\u0018\u0000\u05bf\u05c0\u0001"+
-    "\u0000\u0000\u0000\u05c0\u05c1\u0006\u00c5\u000f\u0000\u05c1\u05c2\u0006"+
-    "\u00c5\u000b\u0000\u05c2\u019a\u0001\u0000\u0000\u0000A\u0000\u0001\u0002"+
-    "\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u024c\u0256\u025a"+
-    "\u025d\u0266\u0268\u0273\u0286\u028b\u0294\u029b\u02a0\u02a2\u02ad\u02b5"+
-    "\u02b8\u02ba\u02bf\u02c4\u02ca\u02d1\u02d6\u02dc\u02df\u02e7\u02eb\u036a"+
-    "\u036f\u0376\u0378\u0388\u038d\u0392\u0394\u039a\u03e7\u03ec\u041b\u041f"+
-    "\u0424\u0429\u042e\u0430\u0434\u0436\u048b\u048f\u0494\u051f\u0521$\u0005"+
-    "\u0001\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005"+
-    "\u0003\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b"+
-    "\u0000\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0013\u0000"+
-    "\u0007A\u0000\u0005\u0000\u0000\u0007\u0019\u0000\u0007B\u0000\u0007h"+
-    "\u0000\u0007\"\u0000\u0007 \u0000\u0007L\u0000\u0007\u001a\u0000\u0007"+
-    "$\u0000\u00070\u0000\u0007@\u0000\u0007P\u0000\u0005\n\u0000\u0005\u0007"+
+    "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001"+
+    "\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001"+
+    "\u008f\u0001\u008f\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001"+
+    "\u0090\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001"+
+    "\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001"+
+    "\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001"+
+    "\u0094\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001"+
+    "\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001\u0097\u0001"+
+    "\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0098\u0001"+
+    "\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u009a\u0001\u009a\u0001"+
+    "\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009b\u0001"+
+    "\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001"+
+    "\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e\u0001\u009e\u0001"+
+    "\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f\u0001"+
+    "\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1\u0001"+
+    "\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0004"+
+    "\u00a2\u0524\b\u00a2\u000b\u00a2\f\u00a2\u0525\u0001\u00a3\u0001\u00a3"+
+    "\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4"+
+    "\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6\u0001\u00a6"+
+    "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7"+
+    "\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9"+
+    "\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001\u00aa\u0001\u00aa"+
+    "\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab"+
+    "\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ad\u0001\u00ad"+
+    "\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae"+
+    "\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0"+
+    "\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b1\u0001\u00b1"+
+    "\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2"+
+    "\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4"+
+    "\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5"+
+    "\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7"+
+    "\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8\u0001\u00b8\u0001\u00b8"+
+    "\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001\u00b9\u0001\u00b9"+
+    "\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001\u00ba"+
+    "\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bc"+
+    "\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001\u00bd"+
+    "\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be\u0001\u00be"+
+    "\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf\u0001\u00bf\u0001\u00bf"+
+    "\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1"+
+    "\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2"+
+    "\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3\u0001\u00c3\u0001\u00c3"+
+    "\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4\u0001\u00c4\u0001\u00c4"+
+    "\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5\u0001\u00c5\u0001\u00c5"+
+    "\u0001\u00c5\u0001\u00c5\u0002\u025f\u02a4\u0000\u00c6\u000f\u0001\u0011"+
+    "\u0002\u0013\u0003\u0015\u0004\u0017\u0005\u0019\u0006\u001b\u0007\u001d"+
+    "\b\u001f\t!\n#\u000b%\f\'\r)\u000e+\u000f-\u0010/\u00111\u00123\u0013"+
+    "5\u00147\u00159\u0016;\u0017=\u0018?\u0000A\u0000C\u0000E\u0000G\u0000"+
+    "I\u0000K\u0000M\u0000O\u0000Q\u0000S\u0019U\u001aW\u001bY\u001c[\u001d"+
+    "]\u001e_\u001fa c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f/\u00810\u00831\u0085"+
+    "2\u00873\u00894\u008b5\u008d6\u008f7\u00918\u00939\u0095:\u0097;\u0099"+
+    "<\u009b=\u009d>\u009f?\u00a1\u0000\u00a3@\u00a5A\u00a7B\u00a9C\u00ab\u0000"+
+    "\u00adD\u00afE\u00b1F\u00b3G\u00b5\u0000\u00b7\u0000\u00b9H\u00bbI\u00bd"+
+    "J\u00bf\u0000\u00c1\u0000\u00c3\u0000\u00c5\u0000\u00c7\u0000\u00c9\u0000"+
+    "\u00cbK\u00cd\u0000\u00cfL\u00d1\u0000\u00d3\u0000\u00d5M\u00d7N\u00d9"+
+    "O\u00db\u0000\u00dd\u0000\u00df\u0000\u00e1\u0000\u00e3\u0000\u00e5\u0000"+
+    "\u00e7\u0000\u00e9P\u00ebQ\u00edR\u00efS\u00f1\u0000\u00f3\u0000\u00f5"+
+    "\u0000\u00f7\u0000\u00f9\u0000\u00fb\u0000\u00fdT\u00ff\u0000\u0101U\u0103"+
+    "V\u0105W\u0107\u0000\u0109\u0000\u010bX\u010dY\u010f\u0000\u0111Z\u0113"+
+    "\u0000\u0115[\u0117\\\u0119]\u011b\u0000\u011d\u0000\u011f\u0000\u0121"+
+    "\u0000\u0123\u0000\u0125\u0000\u0127\u0000\u0129\u0000\u012b\u0000\u012d"+
+    "^\u012f_\u0131`\u0133\u0000\u0135\u0000\u0137\u0000\u0139\u0000\u013b"+
+    "\u0000\u013d\u0000\u013fa\u0141b\u0143c\u0145\u0000\u0147d\u0149e\u014b"+
+    "f\u014dg\u014f\u0000\u0151h\u0153i\u0155j\u0157k\u0159l\u015b\u0000\u015d"+
+    "\u0000\u015f\u0000\u0161\u0000\u0163\u0000\u0165\u0000\u0167\u0000\u0169"+
+    "m\u016bn\u016do\u016f\u0000\u0171\u0000\u0173\u0000\u0175\u0000\u0177"+
+    "p\u0179q\u017br\u017d\u0000\u017f\u0000\u0181\u0000\u0183s\u0185t\u0187"+
+    "u\u0189\u0000\u018b\u0000\u018dv\u018fw\u0191x\u0193\u0000\u0195\u0000"+
+    "\u0197\u0000\u0199\u0000\u000f\u0000\u0001\u0002\u0003\u0004\u0005\u0006"+
+    "\u0007\b\t\n\u000b\f\r\u000e#\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000"+
+    "SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002"+
+    "\u0000OOoo\u0002\u0000PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000"+
+    "VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002"+
+    "\u0000MMmm\u0002\u0000GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000"+
+    "UUuu\u0006\u0000\t\n\r\r  //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r"+
+    "\r  \u0001\u000009\u0002\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000"+
+    "\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000"+
+    "YYyy\u000b\u0000\t\n\r\r  \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000"+
+    "\t\n\r\r  \"#,,//::<<>?\\\\||\u05e3\u0000\u000f\u0001\u0000\u0000\u0000"+
+    "\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000"+
+    "\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000"+
+    "\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000"+
+    "\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000"+
+    "\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%"+
+    "\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001"+
+    "\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000"+
+    "\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u0000"+
+    "3\u0001\u0000\u0000\u0000\u00005\u0001\u0000\u0000\u0000\u00007\u0001"+
+    "\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000\u0000;\u0001\u0000\u0000"+
+    "\u0000\u0001=\u0001\u0000\u0000\u0000\u0001S\u0001\u0000\u0000\u0000\u0001"+
+    "U\u0001\u0000\u0000\u0000\u0001W\u0001\u0000\u0000\u0000\u0001Y\u0001"+
+    "\u0000\u0000\u0000\u0001[\u0001\u0000\u0000\u0000\u0001]\u0001\u0000\u0000"+
+    "\u0000\u0001_\u0001\u0000\u0000\u0000\u0001a\u0001\u0000\u0000\u0000\u0001"+
+    "c\u0001\u0000\u0000\u0000\u0001e\u0001\u0000\u0000\u0000\u0001g\u0001"+
+    "\u0000\u0000\u0000\u0001i\u0001\u0000\u0000\u0000\u0001k\u0001\u0000\u0000"+
+    "\u0000\u0001m\u0001\u0000\u0000\u0000\u0001o\u0001\u0000\u0000\u0000\u0001"+
+    "q\u0001\u0000\u0000\u0000\u0001s\u0001\u0000\u0000\u0000\u0001u\u0001"+
+    "\u0000\u0000\u0000\u0001w\u0001\u0000\u0000\u0000\u0001y\u0001\u0000\u0000"+
+    "\u0000\u0001{\u0001\u0000\u0000\u0000\u0001}\u0001\u0000\u0000\u0000\u0001"+
+    "\u007f\u0001\u0000\u0000\u0000\u0001\u0081\u0001\u0000\u0000\u0000\u0001"+
+    "\u0083\u0001\u0000\u0000\u0000\u0001\u0085\u0001\u0000\u0000\u0000\u0001"+
+    "\u0087\u0001\u0000\u0000\u0000\u0001\u0089\u0001\u0000\u0000\u0000\u0001"+
+    "\u008b\u0001\u0000\u0000\u0000\u0001\u008d\u0001\u0000\u0000\u0000\u0001"+
+    "\u008f\u0001\u0000\u0000\u0000\u0001\u0091\u0001\u0000\u0000\u0000\u0001"+
+    "\u0093\u0001\u0000\u0000\u0000\u0001\u0095\u0001\u0000\u0000\u0000\u0001"+
+    "\u0097\u0001\u0000\u0000\u0000\u0001\u0099\u0001\u0000\u0000\u0000\u0001"+
+    "\u009b\u0001\u0000\u0000\u0000\u0001\u009d\u0001\u0000\u0000\u0000\u0001"+
+    "\u009f\u0001\u0000\u0000\u0000\u0001\u00a1\u0001\u0000\u0000\u0000\u0001"+
+    "\u00a3\u0001\u0000\u0000\u0000\u0001\u00a5\u0001\u0000\u0000\u0000\u0001"+
+    "\u00a7\u0001\u0000\u0000\u0000\u0001\u00a9\u0001\u0000\u0000\u0000\u0001"+
+    "\u00ad\u0001\u0000\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001"+
+    "\u00b1\u0001\u0000\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0002"+
+    "\u00b5\u0001\u0000\u0000\u0000\u0002\u00b7\u0001\u0000\u0000\u0000\u0002"+
+    "\u00b9\u0001\u0000\u0000\u0000\u0002\u00bb\u0001\u0000\u0000\u0000\u0002"+
+    "\u00bd\u0001\u0000\u0000\u0000\u0003\u00bf\u0001\u0000\u0000\u0000\u0003"+
+    "\u00c1\u0001\u0000\u0000\u0000\u0003\u00c3\u0001\u0000\u0000\u0000\u0003"+
+    "\u00c5\u0001\u0000\u0000\u0000\u0003\u00c7\u0001\u0000\u0000\u0000\u0003"+
+    "\u00c9\u0001\u0000\u0000\u0000\u0003\u00cb\u0001\u0000\u0000\u0000\u0003"+
+    "\u00cf\u0001\u0000\u0000\u0000\u0003\u00d1\u0001\u0000\u0000\u0000\u0003"+
+    "\u00d3\u0001\u0000\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003"+
+    "\u00d7\u0001\u0000\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0004"+
+    "\u00db\u0001\u0000\u0000\u0000\u0004\u00dd\u0001\u0000\u0000\u0000\u0004"+
+    "\u00df\u0001\u0000\u0000\u0000\u0004\u00e1\u0001\u0000\u0000\u0000\u0004"+
+    "\u00e3\u0001\u0000\u0000\u0000\u0004\u00e9\u0001\u0000\u0000\u0000\u0004"+
+    "\u00eb\u0001\u0000\u0000\u0000\u0004\u00ed\u0001\u0000\u0000\u0000\u0004"+
+    "\u00ef\u0001\u0000\u0000\u0000\u0005\u00f1\u0001\u0000\u0000\u0000\u0005"+
+    "\u00f3\u0001\u0000\u0000\u0000\u0005\u00f5\u0001\u0000\u0000\u0000\u0005"+
+    "\u00f7\u0001\u0000\u0000\u0000\u0005\u00f9\u0001\u0000\u0000\u0000\u0005"+
+    "\u00fb\u0001\u0000\u0000\u0000\u0005\u00fd\u0001\u0000\u0000\u0000\u0005"+
+    "\u00ff\u0001\u0000\u0000\u0000\u0005\u0101\u0001\u0000\u0000\u0000\u0005"+
+    "\u0103\u0001\u0000\u0000\u0000\u0005\u0105\u0001\u0000\u0000\u0000\u0006"+
+    "\u0107\u0001\u0000\u0000\u0000\u0006\u0109\u0001\u0000\u0000\u0000\u0006"+
+    "\u010b\u0001\u0000\u0000\u0000\u0006\u010d\u0001\u0000\u0000\u0000\u0006"+
+    "\u0111\u0001\u0000\u0000\u0000\u0006\u0113\u0001\u0000\u0000\u0000\u0006"+
+    "\u0115\u0001\u0000\u0000\u0000\u0006\u0117\u0001\u0000\u0000\u0000\u0006"+
+    "\u0119\u0001\u0000\u0000\u0000\u0007\u011b\u0001\u0000\u0000\u0000\u0007"+
+    "\u011d\u0001\u0000\u0000\u0000\u0007\u011f\u0001\u0000\u0000\u0000\u0007"+
+    "\u0121\u0001\u0000\u0000\u0000\u0007\u0123\u0001\u0000\u0000\u0000\u0007"+
+    "\u0125\u0001\u0000\u0000\u0000\u0007\u0127\u0001\u0000\u0000\u0000\u0007"+
+    "\u0129\u0001\u0000\u0000\u0000\u0007\u012b\u0001\u0000\u0000\u0000\u0007"+
+    "\u012d\u0001\u0000\u0000\u0000\u0007\u012f\u0001\u0000\u0000\u0000\u0007"+
+    "\u0131\u0001\u0000\u0000\u0000\b\u0133\u0001\u0000\u0000\u0000\b\u0135"+
+    "\u0001\u0000\u0000\u0000\b\u0137\u0001\u0000\u0000\u0000\b\u0139\u0001"+
+    "\u0000\u0000\u0000\b\u013b\u0001\u0000\u0000\u0000\b\u013d\u0001\u0000"+
+    "\u0000\u0000\b\u013f\u0001\u0000\u0000\u0000\b\u0141\u0001\u0000\u0000"+
+    "\u0000\b\u0143\u0001\u0000\u0000\u0000\t\u0145\u0001\u0000\u0000\u0000"+
+    "\t\u0147\u0001\u0000\u0000\u0000\t\u0149\u0001\u0000\u0000\u0000\t\u014b"+
+    "\u0001\u0000\u0000\u0000\t\u014d\u0001\u0000\u0000\u0000\n\u014f\u0001"+
+    "\u0000\u0000\u0000\n\u0151\u0001\u0000\u0000\u0000\n\u0153\u0001\u0000"+
+    "\u0000\u0000\n\u0155\u0001\u0000\u0000\u0000\n\u0157\u0001\u0000\u0000"+
+    "\u0000\n\u0159\u0001\u0000\u0000\u0000\u000b\u015b\u0001\u0000\u0000\u0000"+
+    "\u000b\u015d\u0001\u0000\u0000\u0000\u000b\u015f\u0001\u0000\u0000\u0000"+
+    "\u000b\u0161\u0001\u0000\u0000\u0000\u000b\u0163\u0001\u0000\u0000\u0000"+
+    "\u000b\u0165\u0001\u0000\u0000\u0000\u000b\u0167\u0001\u0000\u0000\u0000"+
+    "\u000b\u0169\u0001\u0000\u0000\u0000\u000b\u016b\u0001\u0000\u0000\u0000"+
+    "\u000b\u016d\u0001\u0000\u0000\u0000\f\u016f\u0001\u0000\u0000\u0000\f"+
+    "\u0171\u0001\u0000\u0000\u0000\f\u0173\u0001\u0000\u0000\u0000\f\u0175"+
+    "\u0001\u0000\u0000\u0000\f\u0177\u0001\u0000\u0000\u0000\f\u0179\u0001"+
+    "\u0000\u0000\u0000\f\u017b\u0001\u0000\u0000\u0000\r\u017d\u0001\u0000"+
+    "\u0000\u0000\r\u017f\u0001\u0000\u0000\u0000\r\u0181\u0001\u0000\u0000"+
+    "\u0000\r\u0183\u0001\u0000\u0000\u0000\r\u0185\u0001\u0000\u0000\u0000"+
+    "\r\u0187\u0001\u0000\u0000\u0000\u000e\u0189\u0001\u0000\u0000\u0000\u000e"+
+    "\u018b\u0001\u0000\u0000\u0000\u000e\u018d\u0001\u0000\u0000\u0000\u000e"+
+    "\u018f\u0001\u0000\u0000\u0000\u000e\u0191\u0001\u0000\u0000\u0000\u000e"+
+    "\u0193\u0001\u0000\u0000\u0000\u000e\u0195\u0001\u0000\u0000\u0000\u000e"+
+    "\u0197\u0001\u0000\u0000\u0000\u000e\u0199\u0001\u0000\u0000\u0000\u000f"+
+    "\u019b\u0001\u0000\u0000\u0000\u0011\u01a5\u0001\u0000\u0000\u0000\u0013"+
+    "\u01ac\u0001\u0000\u0000\u0000\u0015\u01b5\u0001\u0000\u0000\u0000\u0017"+
+    "\u01bc\u0001\u0000\u0000\u0000\u0019\u01c6\u0001\u0000\u0000\u0000\u001b"+
+    "\u01cd\u0001\u0000\u0000\u0000\u001d\u01d4\u0001\u0000\u0000\u0000\u001f"+
+    "\u01db\u0001\u0000\u0000\u0000!\u01e3\u0001\u0000\u0000\u0000#\u01ef\u0001"+
+    "\u0000\u0000\u0000%\u01f8\u0001\u0000\u0000\u0000\'\u01fe\u0001\u0000"+
+    "\u0000\u0000)\u0205\u0001\u0000\u0000\u0000+\u020c\u0001\u0000\u0000\u0000"+
+    "-\u0214\u0001\u0000\u0000\u0000/\u021c\u0001\u0000\u0000\u00001\u022b"+
+    "\u0001\u0000\u0000\u00003\u0235\u0001\u0000\u0000\u00005\u0241\u0001\u0000"+
+    "\u0000\u00007\u0247\u0001\u0000\u0000\u00009\u0258\u0001\u0000\u0000\u0000"+
+    ";\u0268\u0001\u0000\u0000\u0000=\u026e\u0001\u0000\u0000\u0000?\u0272"+
+    "\u0001\u0000\u0000\u0000A\u0274\u0001\u0000\u0000\u0000C\u0276\u0001\u0000"+
+    "\u0000\u0000E\u0279\u0001\u0000\u0000\u0000G\u027b\u0001\u0000\u0000\u0000"+
+    "I\u0284\u0001\u0000\u0000\u0000K\u0286\u0001\u0000\u0000\u0000M\u028b"+
+    "\u0001\u0000\u0000\u0000O\u028d\u0001\u0000\u0000\u0000Q\u0292\u0001\u0000"+
+    "\u0000\u0000S\u02b1\u0001\u0000\u0000\u0000U\u02b4\u0001\u0000\u0000\u0000"+
+    "W\u02e2\u0001\u0000\u0000\u0000Y\u02e4\u0001\u0000\u0000\u0000[\u02e7"+
+    "\u0001\u0000\u0000\u0000]\u02eb\u0001\u0000\u0000\u0000_\u02ef\u0001\u0000"+
+    "\u0000\u0000a\u02f1\u0001\u0000\u0000\u0000c\u02f4\u0001\u0000\u0000\u0000"+
+    "e\u02f6\u0001\u0000\u0000\u0000g\u02fb\u0001\u0000\u0000\u0000i\u02fd"+
+    "\u0001\u0000\u0000\u0000k\u0303\u0001\u0000\u0000\u0000m\u0309\u0001\u0000"+
+    "\u0000\u0000o\u030c\u0001\u0000\u0000\u0000q\u030f\u0001\u0000\u0000\u0000"+
+    "s\u0314\u0001\u0000\u0000\u0000u\u0319\u0001\u0000\u0000\u0000w\u031b"+
+    "\u0001\u0000\u0000\u0000y\u031f\u0001\u0000\u0000\u0000{\u0324\u0001\u0000"+
+    "\u0000\u0000}\u032a\u0001\u0000\u0000\u0000\u007f\u032d\u0001\u0000\u0000"+
+    "\u0000\u0081\u032f\u0001\u0000\u0000\u0000\u0083\u0335\u0001\u0000\u0000"+
+    "\u0000\u0085\u0337\u0001\u0000\u0000\u0000\u0087\u033c\u0001\u0000\u0000"+
+    "\u0000\u0089\u033f\u0001\u0000\u0000\u0000\u008b\u0342\u0001\u0000\u0000"+
+    "\u0000\u008d\u0345\u0001\u0000\u0000\u0000\u008f\u0347\u0001\u0000\u0000"+
+    "\u0000\u0091\u034a\u0001\u0000\u0000\u0000\u0093\u034c\u0001\u0000\u0000"+
+    "\u0000\u0095\u034f\u0001\u0000\u0000\u0000\u0097\u0351\u0001\u0000\u0000"+
+    "\u0000\u0099\u0353\u0001\u0000\u0000\u0000\u009b\u0355\u0001\u0000\u0000"+
+    "\u0000\u009d\u0357\u0001\u0000\u0000\u0000\u009f\u0359\u0001\u0000\u0000"+
+    "\u0000\u00a1\u035f\u0001\u0000\u0000\u0000\u00a3\u0374\u0001\u0000\u0000"+
+    "\u0000\u00a5\u0376\u0001\u0000\u0000\u0000\u00a7\u037b\u0001\u0000\u0000"+
+    "\u0000\u00a9\u0390\u0001\u0000\u0000\u0000\u00ab\u0392\u0001\u0000\u0000"+
+    "\u0000\u00ad\u039a\u0001\u0000\u0000\u0000\u00af\u039c\u0001\u0000\u0000"+
+    "\u0000\u00b1\u03a0\u0001\u0000\u0000\u0000\u00b3\u03a4\u0001\u0000\u0000"+
+    "\u0000\u00b5\u03a8\u0001\u0000\u0000\u0000\u00b7\u03ad\u0001\u0000\u0000"+
+    "\u0000\u00b9\u03b2\u0001\u0000\u0000\u0000\u00bb\u03b6\u0001\u0000\u0000"+
+    "\u0000\u00bd\u03ba\u0001\u0000\u0000\u0000\u00bf\u03be\u0001\u0000\u0000"+
+    "\u0000\u00c1\u03c3\u0001\u0000\u0000\u0000\u00c3\u03c7\u0001\u0000\u0000"+
+    "\u0000\u00c5\u03cb\u0001\u0000\u0000\u0000\u00c7\u03cf\u0001\u0000\u0000"+
+    "\u0000\u00c9\u03d3\u0001\u0000\u0000\u0000\u00cb\u03d7\u0001\u0000\u0000"+
+    "\u0000\u00cd\u03e3\u0001\u0000\u0000\u0000\u00cf\u03e6\u0001\u0000\u0000"+
+    "\u0000\u00d1\u03ea\u0001\u0000\u0000\u0000\u00d3\u03ee\u0001\u0000\u0000"+
+    "\u0000\u00d5\u03f2\u0001\u0000\u0000\u0000\u00d7\u03f6\u0001\u0000\u0000"+
+    "\u0000\u00d9\u03fa\u0001\u0000\u0000\u0000\u00db\u03fe\u0001\u0000\u0000"+
+    "\u0000\u00dd\u0403\u0001\u0000\u0000\u0000\u00df\u0407\u0001\u0000\u0000"+
+    "\u0000\u00e1\u040b\u0001\u0000\u0000\u0000\u00e3\u0410\u0001\u0000\u0000"+
+    "\u0000\u00e5\u0419\u0001\u0000\u0000\u0000\u00e7\u042e\u0001\u0000\u0000"+
+    "\u0000\u00e9\u0432\u0001\u0000\u0000\u0000\u00eb\u0436\u0001\u0000\u0000"+
+    "\u0000\u00ed\u043a\u0001\u0000\u0000\u0000\u00ef\u043e\u0001\u0000\u0000"+
+    "\u0000\u00f1\u0442\u0001\u0000\u0000\u0000\u00f3\u0447\u0001\u0000\u0000"+
+    "\u0000\u00f5\u044b\u0001\u0000\u0000\u0000\u00f7\u044f\u0001\u0000\u0000"+
+    "\u0000\u00f9\u0453\u0001\u0000\u0000\u0000\u00fb\u0458\u0001\u0000\u0000"+
+    "\u0000\u00fd\u045d\u0001\u0000\u0000\u0000\u00ff\u0460\u0001\u0000\u0000"+
+    "\u0000\u0101\u0464\u0001\u0000\u0000\u0000\u0103\u0468\u0001\u0000\u0000"+
+    "\u0000\u0105\u046c\u0001\u0000\u0000\u0000\u0107\u0470\u0001\u0000\u0000"+
+    "\u0000\u0109\u0475\u0001\u0000\u0000\u0000\u010b\u047a\u0001\u0000\u0000"+
+    "\u0000\u010d\u047f\u0001\u0000\u0000\u0000\u010f\u0486\u0001\u0000\u0000"+
+    "\u0000\u0111\u048f\u0001\u0000\u0000\u0000\u0113\u0496\u0001\u0000\u0000"+
+    "\u0000\u0115\u049a\u0001\u0000\u0000\u0000\u0117\u049e\u0001\u0000\u0000"+
+    "\u0000\u0119\u04a2\u0001\u0000\u0000\u0000\u011b\u04a6\u0001\u0000\u0000"+
+    "\u0000\u011d\u04ac\u0001\u0000\u0000\u0000\u011f\u04b0\u0001\u0000\u0000"+
+    "\u0000\u0121\u04b4\u0001\u0000\u0000\u0000\u0123\u04b8\u0001\u0000\u0000"+
+    "\u0000\u0125\u04bc\u0001\u0000\u0000\u0000\u0127\u04c0\u0001\u0000\u0000"+
+    "\u0000\u0129\u04c4\u0001\u0000\u0000\u0000\u012b\u04c9\u0001\u0000\u0000"+
+    "\u0000\u012d\u04ce\u0001\u0000\u0000\u0000\u012f\u04d2\u0001\u0000\u0000"+
+    "\u0000\u0131\u04d6\u0001\u0000\u0000\u0000\u0133\u04da\u0001\u0000\u0000"+
+    "\u0000\u0135\u04df\u0001\u0000\u0000\u0000\u0137\u04e3\u0001\u0000\u0000"+
+    "\u0000\u0139\u04e8\u0001\u0000\u0000\u0000\u013b\u04ed\u0001\u0000\u0000"+
+    "\u0000\u013d\u04f1\u0001\u0000\u0000\u0000\u013f\u04f5\u0001\u0000\u0000"+
+    "\u0000\u0141\u04f9\u0001\u0000\u0000\u0000\u0143\u04fd\u0001\u0000\u0000"+
+    "\u0000\u0145\u0501\u0001\u0000\u0000\u0000\u0147\u0506\u0001\u0000\u0000"+
+    "\u0000\u0149\u050b\u0001\u0000\u0000\u0000\u014b\u050f\u0001\u0000\u0000"+
+    "\u0000\u014d\u0513\u0001\u0000\u0000\u0000\u014f\u0517\u0001\u0000\u0000"+
+    "\u0000\u0151\u051c\u0001\u0000\u0000\u0000\u0153\u0523\u0001\u0000\u0000"+
+    "\u0000\u0155\u0527\u0001\u0000\u0000\u0000\u0157\u052b\u0001\u0000\u0000"+
+    "\u0000\u0159\u052f\u0001\u0000\u0000\u0000\u015b\u0533\u0001\u0000\u0000"+
+    "\u0000\u015d\u0538\u0001\u0000\u0000\u0000\u015f\u053c\u0001\u0000\u0000"+
+    "\u0000\u0161\u0540\u0001\u0000\u0000\u0000\u0163\u0544\u0001\u0000\u0000"+
+    "\u0000\u0165\u0549\u0001\u0000\u0000\u0000\u0167\u054d\u0001\u0000\u0000"+
+    "\u0000\u0169\u0551\u0001\u0000\u0000\u0000\u016b\u0555\u0001\u0000\u0000"+
+    "\u0000\u016d\u0559\u0001\u0000\u0000\u0000\u016f\u055d\u0001\u0000\u0000"+
+    "\u0000\u0171\u0563\u0001\u0000\u0000\u0000\u0173\u0567\u0001\u0000\u0000"+
+    "\u0000\u0175\u056b\u0001\u0000\u0000\u0000\u0177\u056f\u0001\u0000\u0000"+
+    "\u0000\u0179\u0573\u0001\u0000\u0000\u0000\u017b\u0577\u0001\u0000\u0000"+
+    "\u0000\u017d\u057b\u0001\u0000\u0000\u0000\u017f\u0580\u0001\u0000\u0000"+
+    "\u0000\u0181\u0586\u0001\u0000\u0000\u0000\u0183\u058c\u0001\u0000\u0000"+
+    "\u0000\u0185\u0590\u0001\u0000\u0000\u0000\u0187\u0594\u0001\u0000\u0000"+
+    "\u0000\u0189\u0598\u0001\u0000\u0000\u0000\u018b\u059e\u0001\u0000\u0000"+
+    "\u0000\u018d\u05a4\u0001\u0000\u0000\u0000\u018f\u05a8\u0001\u0000\u0000"+
+    "\u0000\u0191\u05ac\u0001\u0000\u0000\u0000\u0193\u05b0\u0001\u0000\u0000"+
+    "\u0000\u0195\u05b6\u0001\u0000\u0000\u0000\u0197\u05bc\u0001\u0000\u0000"+
+    "\u0000\u0199\u05c2\u0001\u0000\u0000\u0000\u019b\u019c\u0007\u0000\u0000"+
+    "\u0000\u019c\u019d\u0007\u0001\u0000\u0000\u019d\u019e\u0007\u0002\u0000"+
+    "\u0000\u019e\u019f\u0007\u0002\u0000\u0000\u019f\u01a0\u0007\u0003\u0000"+
+    "\u0000\u01a0\u01a1\u0007\u0004\u0000\u0000\u01a1\u01a2\u0007\u0005\u0000"+
+    "\u0000\u01a2\u01a3\u0001\u0000\u0000\u0000\u01a3\u01a4\u0006\u0000\u0000"+
+    "\u0000\u01a4\u0010\u0001\u0000\u0000\u0000\u01a5\u01a6\u0007\u0000\u0000"+
+    "\u0000\u01a6\u01a7\u0007\u0006\u0000\u0000\u01a7\u01a8\u0007\u0007\u0000"+
+    "\u0000\u01a8\u01a9\u0007\b\u0000\u0000\u01a9\u01aa\u0001\u0000\u0000\u0000"+
+    "\u01aa\u01ab\u0006\u0001\u0001\u0000\u01ab\u0012\u0001\u0000\u0000\u0000"+
+    "\u01ac\u01ad\u0007\u0003\u0000\u0000\u01ad\u01ae\u0007\t\u0000\u0000\u01ae"+
+    "\u01af\u0007\u0006\u0000\u0000\u01af\u01b0\u0007\u0001\u0000\u0000\u01b0"+
+    "\u01b1\u0007\u0004\u0000\u0000\u01b1\u01b2\u0007\n\u0000\u0000\u01b2\u01b3"+
+    "\u0001\u0000\u0000\u0000\u01b3\u01b4\u0006\u0002\u0002\u0000\u01b4\u0014"+
+    "\u0001\u0000\u0000\u0000\u01b5\u01b6\u0007\u0003\u0000\u0000\u01b6\u01b7"+
+    "\u0007\u000b\u0000\u0000\u01b7\u01b8\u0007\f\u0000\u0000\u01b8\u01b9\u0007"+
+    "\r\u0000\u0000\u01b9\u01ba\u0001\u0000\u0000\u0000\u01ba\u01bb\u0006\u0003"+
+    "\u0000\u0000\u01bb\u0016\u0001\u0000\u0000\u0000\u01bc\u01bd\u0007\u0003"+
+    "\u0000\u0000\u01bd\u01be\u0007\u000e\u0000\u0000\u01be\u01bf\u0007\b\u0000"+
+    "\u0000\u01bf\u01c0\u0007\r\u0000\u0000\u01c0\u01c1\u0007\f\u0000\u0000"+
+    "\u01c1\u01c2\u0007\u0001\u0000\u0000\u01c2\u01c3\u0007\t\u0000\u0000\u01c3"+
+    "\u01c4\u0001\u0000\u0000\u0000\u01c4\u01c5\u0006\u0004\u0003\u0000\u01c5"+
+    "\u0018\u0001\u0000\u0000\u0000\u01c6\u01c7\u0007\u000f\u0000\u0000\u01c7"+
+    "\u01c8\u0007\u0006\u0000\u0000\u01c8\u01c9\u0007\u0007\u0000\u0000\u01c9"+
+    "\u01ca\u0007\u0010\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb"+
+    "\u01cc\u0006\u0005\u0004\u0000\u01cc\u001a\u0001\u0000\u0000\u0000\u01cd"+
+    "\u01ce\u0007\u0011\u0000\u0000\u01ce\u01cf\u0007\u0006\u0000\u0000\u01cf"+
+    "\u01d0\u0007\u0007\u0000\u0000\u01d0\u01d1\u0007\u0012\u0000\u0000\u01d1"+
+    "\u01d2\u0001\u0000\u0000\u0000\u01d2\u01d3\u0006\u0006\u0000\u0000\u01d3"+
+    "\u001c\u0001\u0000\u0000\u0000\u01d4\u01d5\u0007\u0012\u0000\u0000\u01d5"+
+    "\u01d6\u0007\u0003\u0000\u0000\u01d6\u01d7\u0007\u0003\u0000\u0000\u01d7"+
+    "\u01d8\u0007\b\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da"+
+    "\u0006\u0007\u0001\u0000\u01da\u001e\u0001\u0000\u0000\u0000\u01db\u01dc"+
+    "\u0007\r\u0000\u0000\u01dc\u01dd\u0007\u0001\u0000\u0000\u01dd\u01de\u0007"+
+    "\u0010\u0000\u0000\u01de\u01df\u0007\u0001\u0000\u0000\u01df\u01e0\u0007"+
+    "\u0005\u0000\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e1\u01e2\u0006"+
+    "\b\u0000\u0000\u01e2 \u0001\u0000\u0000\u0000\u01e3\u01e4\u0007\u0010"+
+    "\u0000\u0000\u01e4\u01e5\u0007\u000b\u0000\u0000\u01e5\u01e6\u0005_\u0000"+
+    "\u0000\u01e6\u01e7\u0007\u0003\u0000\u0000\u01e7\u01e8\u0007\u000e\u0000"+
+    "\u0000\u01e8\u01e9\u0007\b\u0000\u0000\u01e9\u01ea\u0007\f\u0000\u0000"+
+    "\u01ea\u01eb\u0007\t\u0000\u0000\u01eb\u01ec\u0007\u0000\u0000\u0000\u01ec"+
+    "\u01ed\u0001\u0000\u0000\u0000\u01ed\u01ee\u0006\t\u0005\u0000\u01ee\""+
+    "\u0001\u0000\u0000\u0000\u01ef\u01f0\u0007\u0006\u0000\u0000\u01f0\u01f1"+
+    "\u0007\u0003\u0000\u0000\u01f1\u01f2\u0007\t\u0000\u0000\u01f2\u01f3\u0007"+
+    "\f\u0000\u0000\u01f3\u01f4\u0007\u0010\u0000\u0000\u01f4\u01f5\u0007\u0003"+
+    "\u0000\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0006\n\u0006"+
+    "\u0000\u01f7$\u0001\u0000\u0000\u0000\u01f8\u01f9\u0007\u0006\u0000\u0000"+
+    "\u01f9\u01fa\u0007\u0007\u0000\u0000\u01fa\u01fb\u0007\u0013\u0000\u0000"+
+    "\u01fb\u01fc\u0001\u0000\u0000\u0000\u01fc\u01fd\u0006\u000b\u0000\u0000"+
+    "\u01fd&\u0001\u0000\u0000\u0000\u01fe\u01ff\u0007\u0002\u0000\u0000\u01ff"+
+    "\u0200\u0007\n\u0000\u0000\u0200\u0201\u0007\u0007\u0000\u0000\u0201\u0202"+
+    "\u0007\u0013\u0000\u0000\u0202\u0203\u0001\u0000\u0000\u0000\u0203\u0204"+
+    "\u0006\f\u0007\u0000\u0204(\u0001\u0000\u0000\u0000\u0205\u0206\u0007"+
+    "\u0002\u0000\u0000\u0206\u0207\u0007\u0007\u0000\u0000\u0207\u0208\u0007"+
+    "\u0006\u0000\u0000\u0208\u0209\u0007\u0005\u0000\u0000\u0209\u020a\u0001"+
+    "\u0000\u0000\u0000\u020a\u020b\u0006\r\u0000\u0000\u020b*\u0001\u0000"+
+    "\u0000\u0000\u020c\u020d\u0007\u0002\u0000\u0000\u020d\u020e\u0007\u0005"+
+    "\u0000\u0000\u020e\u020f\u0007\f\u0000\u0000\u020f\u0210\u0007\u0005\u0000"+
+    "\u0000\u0210\u0211\u0007\u0002\u0000\u0000\u0211\u0212\u0001\u0000\u0000"+
+    "\u0000\u0212\u0213\u0006\u000e\u0000\u0000\u0213,\u0001\u0000\u0000\u0000"+
+    "\u0214\u0215\u0007\u0013\u0000\u0000\u0215\u0216\u0007\n\u0000\u0000\u0216"+
+    "\u0217\u0007\u0003\u0000\u0000\u0217\u0218\u0007\u0006\u0000\u0000\u0218"+
+    "\u0219\u0007\u0003\u0000\u0000\u0219\u021a\u0001\u0000\u0000\u0000\u021a"+
+    "\u021b\u0006\u000f\u0000\u0000\u021b.\u0001\u0000\u0000\u0000\u021c\u021d"+
+    "\u0004\u0010\u0000\u0000\u021d\u021e\u0007\u0001\u0000\u0000\u021e\u021f"+
+    "\u0007\t\u0000\u0000\u021f\u0220\u0007\r\u0000\u0000\u0220\u0221\u0007"+
+    "\u0001\u0000\u0000\u0221\u0222\u0007\t\u0000\u0000\u0222\u0223\u0007\u0003"+
+    "\u0000\u0000\u0223\u0224\u0007\u0002\u0000\u0000\u0224\u0225\u0007\u0005"+
+    "\u0000\u0000\u0225\u0226\u0007\f\u0000\u0000\u0226\u0227\u0007\u0005\u0000"+
+    "\u0000\u0227\u0228\u0007\u0002\u0000\u0000\u0228\u0229\u0001\u0000\u0000"+
+    "\u0000\u0229\u022a\u0006\u0010\u0000\u0000\u022a0\u0001\u0000\u0000\u0000"+
+    "\u022b\u022c\u0004\u0011\u0001\u0000\u022c\u022d\u0007\r\u0000\u0000\u022d"+
+    "\u022e\u0007\u0007\u0000\u0000\u022e\u022f\u0007\u0007\u0000\u0000\u022f"+
+    "\u0230\u0007\u0012\u0000\u0000\u0230\u0231\u0007\u0014\u0000\u0000\u0231"+
+    "\u0232\u0007\b\u0000\u0000\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234"+
+    "\u0006\u0011\b\u0000\u02342\u0001\u0000\u0000\u0000\u0235\u0236\u0004"+
+    "\u0012\u0002\u0000\u0236\u0237\u0007\u0010\u0000\u0000\u0237\u0238\u0007"+
+    "\u0003\u0000\u0000\u0238\u0239\u0007\u0005\u0000\u0000\u0239\u023a\u0007"+
+    "\u0006\u0000\u0000\u023a\u023b\u0007\u0001\u0000\u0000\u023b\u023c\u0007"+
+    "\u0004\u0000\u0000\u023c\u023d\u0007\u0002\u0000\u0000\u023d\u023e\u0001"+
+    "\u0000\u0000\u0000\u023e\u023f\u0006\u0012\t\u0000\u023f4\u0001\u0000"+
+    "\u0000\u0000\u0240\u0242\b\u0015\u0000\u0000\u0241\u0240\u0001\u0000\u0000"+
+    "\u0000\u0242\u0243\u0001\u0000\u0000\u0000\u0243\u0241\u0001\u0000\u0000"+
+    "\u0000\u0243\u0244\u0001\u0000\u0000\u0000\u0244\u0245\u0001\u0000\u0000"+
+    "\u0000\u0245\u0246\u0006\u0013\u0000\u0000\u02466\u0001\u0000\u0000\u0000"+
+    "\u0247\u0248\u0005/\u0000\u0000\u0248\u0249\u0005/\u0000\u0000\u0249\u024d"+
+    "\u0001\u0000\u0000\u0000\u024a\u024c\b\u0016\u0000\u0000\u024b\u024a\u0001"+
+    "\u0000\u0000\u0000\u024c\u024f\u0001\u0000\u0000\u0000\u024d\u024b\u0001"+
+    "\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u0251\u0001"+
+    "\u0000\u0000\u0000\u024f\u024d\u0001\u0000\u0000\u0000\u0250\u0252\u0005"+
+    "\r\u0000\u0000\u0251\u0250\u0001\u0000\u0000\u0000\u0251\u0252\u0001\u0000"+
+    "\u0000\u0000\u0252\u0254\u0001\u0000\u0000\u0000\u0253\u0255\u0005\n\u0000"+
+    "\u0000\u0254\u0253\u0001\u0000\u0000\u0000\u0254\u0255\u0001\u0000\u0000"+
+    "\u0000\u0255\u0256\u0001\u0000\u0000\u0000\u0256\u0257\u0006\u0014\n\u0000"+
+    "\u02578\u0001\u0000\u0000\u0000\u0258\u0259\u0005/\u0000\u0000\u0259\u025a"+
+    "\u0005*\u0000\u0000\u025a\u025f\u0001\u0000\u0000\u0000\u025b\u025e\u0003"+
+    "9\u0015\u0000\u025c\u025e\t\u0000\u0000\u0000\u025d\u025b\u0001\u0000"+
+    "\u0000\u0000\u025d\u025c\u0001\u0000\u0000\u0000\u025e\u0261\u0001\u0000"+
+    "\u0000\u0000\u025f\u0260\u0001\u0000\u0000\u0000\u025f\u025d\u0001\u0000"+
+    "\u0000\u0000\u0260\u0262\u0001\u0000\u0000\u0000\u0261\u025f\u0001\u0000"+
+    "\u0000\u0000\u0262\u0263\u0005*\u0000\u0000\u0263\u0264\u0005/\u0000\u0000"+
+    "\u0264\u0265\u0001\u0000\u0000\u0000\u0265\u0266\u0006\u0015\n\u0000\u0266"+
+    ":\u0001\u0000\u0000\u0000\u0267\u0269\u0007\u0017\u0000\u0000\u0268\u0267"+
+    "\u0001\u0000\u0000\u0000\u0269\u026a\u0001\u0000\u0000\u0000\u026a\u0268"+
+    "\u0001\u0000\u0000\u0000\u026a\u026b\u0001\u0000\u0000\u0000\u026b\u026c"+
+    "\u0001\u0000\u0000\u0000\u026c\u026d\u0006\u0016\n\u0000\u026d<\u0001"+
+    "\u0000\u0000\u0000\u026e\u026f\u0005|\u0000\u0000\u026f\u0270\u0001\u0000"+
+    "\u0000\u0000\u0270\u0271\u0006\u0017\u000b\u0000\u0271>\u0001\u0000\u0000"+
+    "\u0000\u0272\u0273\u0007\u0018\u0000\u0000\u0273@\u0001\u0000\u0000\u0000"+
+    "\u0274\u0275\u0007\u0019\u0000\u0000\u0275B\u0001\u0000\u0000\u0000\u0276"+
+    "\u0277\u0005\\\u0000\u0000\u0277\u0278\u0007\u001a\u0000\u0000\u0278D"+
+    "\u0001\u0000\u0000\u0000\u0279\u027a\b\u001b\u0000\u0000\u027aF\u0001"+
+    "\u0000\u0000\u0000\u027b\u027d\u0007\u0003\u0000\u0000\u027c\u027e\u0007"+
+    "\u001c\u0000\u0000\u027d\u027c\u0001\u0000\u0000\u0000\u027d\u027e\u0001"+
+    "\u0000\u0000\u0000\u027e\u0280\u0001\u0000\u0000\u0000\u027f\u0281\u0003"+
+    "?\u0018\u0000\u0280\u027f\u0001\u0000\u0000\u0000\u0281\u0282\u0001\u0000"+
+    "\u0000\u0000\u0282\u0280\u0001\u0000\u0000\u0000\u0282\u0283\u0001\u0000"+
+    "\u0000\u0000\u0283H\u0001\u0000\u0000\u0000\u0284\u0285\u0005@\u0000\u0000"+
+    "\u0285J\u0001\u0000\u0000\u0000\u0286\u0287\u0005`\u0000\u0000\u0287L"+
+    "\u0001\u0000\u0000\u0000\u0288\u028c\b\u001d\u0000\u0000\u0289\u028a\u0005"+
+    "`\u0000\u0000\u028a\u028c\u0005`\u0000\u0000\u028b\u0288\u0001\u0000\u0000"+
+    "\u0000\u028b\u0289\u0001\u0000\u0000\u0000\u028cN\u0001\u0000\u0000\u0000"+
+    "\u028d\u028e\u0005_\u0000\u0000\u028eP\u0001\u0000\u0000\u0000\u028f\u0293"+
+    "\u0003A\u0019\u0000\u0290\u0293\u0003?\u0018\u0000\u0291\u0293\u0003O"+
+    " \u0000\u0292\u028f\u0001\u0000\u0000\u0000\u0292\u0290\u0001\u0000\u0000"+
+    "\u0000\u0292\u0291\u0001\u0000\u0000\u0000\u0293R\u0001\u0000\u0000\u0000"+
+    "\u0294\u0299\u0005\"\u0000\u0000\u0295\u0298\u0003C\u001a\u0000\u0296"+
+    "\u0298\u0003E\u001b\u0000\u0297\u0295\u0001\u0000\u0000\u0000\u0297\u0296"+
+    "\u0001\u0000\u0000\u0000\u0298\u029b\u0001\u0000\u0000\u0000\u0299\u0297"+
+    "\u0001\u0000\u0000\u0000\u0299\u029a\u0001\u0000\u0000\u0000\u029a\u029c"+
+    "\u0001\u0000\u0000\u0000\u029b\u0299\u0001\u0000\u0000\u0000\u029c\u02b2"+
+    "\u0005\"\u0000\u0000\u029d\u029e\u0005\"\u0000\u0000\u029e\u029f\u0005"+
+    "\"\u0000\u0000\u029f\u02a0\u0005\"\u0000\u0000\u02a0\u02a4\u0001\u0000"+
+    "\u0000\u0000\u02a1\u02a3\b\u0016\u0000\u0000\u02a2\u02a1\u0001\u0000\u0000"+
+    "\u0000\u02a3\u02a6\u0001\u0000\u0000\u0000\u02a4\u02a5\u0001\u0000\u0000"+
+    "\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5\u02a7\u0001\u0000\u0000"+
+    "\u0000\u02a6\u02a4\u0001\u0000\u0000\u0000\u02a7\u02a8\u0005\"\u0000\u0000"+
+    "\u02a8\u02a9\u0005\"\u0000\u0000\u02a9\u02aa\u0005\"\u0000\u0000\u02aa"+
+    "\u02ac\u0001\u0000\u0000\u0000\u02ab\u02ad\u0005\"\u0000\u0000\u02ac\u02ab"+
+    "\u0001\u0000\u0000\u0000\u02ac\u02ad\u0001\u0000\u0000\u0000\u02ad\u02af"+
+    "\u0001\u0000\u0000\u0000\u02ae\u02b0\u0005\"\u0000\u0000\u02af\u02ae\u0001"+
+    "\u0000\u0000\u0000\u02af\u02b0\u0001\u0000\u0000\u0000\u02b0\u02b2\u0001"+
+    "\u0000\u0000\u0000\u02b1\u0294\u0001\u0000\u0000\u0000\u02b1\u029d\u0001"+
+    "\u0000\u0000\u0000\u02b2T\u0001\u0000\u0000\u0000\u02b3\u02b5\u0003?\u0018"+
+    "\u0000\u02b4\u02b3\u0001\u0000\u0000\u0000\u02b5\u02b6\u0001\u0000\u0000"+
+    "\u0000\u02b6\u02b4\u0001\u0000\u0000\u0000\u02b6\u02b7\u0001\u0000\u0000"+
+    "\u0000\u02b7V\u0001\u0000\u0000\u0000\u02b8\u02ba\u0003?\u0018\u0000\u02b9"+
+    "\u02b8\u0001\u0000\u0000\u0000\u02ba\u02bb\u0001\u0000\u0000\u0000\u02bb"+
+    "\u02b9\u0001\u0000\u0000\u0000\u02bb\u02bc\u0001\u0000\u0000\u0000\u02bc"+
+    "\u02bd\u0001\u0000\u0000\u0000\u02bd\u02c1\u0003g,\u0000\u02be\u02c0\u0003"+
+    "?\u0018\u0000\u02bf\u02be\u0001\u0000\u0000\u0000\u02c0\u02c3\u0001\u0000"+
+    "\u0000\u0000\u02c1\u02bf\u0001\u0000\u0000\u0000\u02c1\u02c2\u0001\u0000"+
+    "\u0000\u0000\u02c2\u02e3\u0001\u0000\u0000\u0000\u02c3\u02c1\u0001\u0000"+
+    "\u0000\u0000\u02c4\u02c6\u0003g,\u0000\u02c5\u02c7\u0003?\u0018\u0000"+
+    "\u02c6\u02c5\u0001\u0000\u0000\u0000\u02c7\u02c8\u0001\u0000\u0000\u0000"+
+    "\u02c8\u02c6\u0001\u0000\u0000\u0000\u02c8\u02c9\u0001\u0000\u0000\u0000"+
+    "\u02c9\u02e3\u0001\u0000\u0000\u0000\u02ca\u02cc\u0003?\u0018\u0000\u02cb"+
+    "\u02ca\u0001\u0000\u0000\u0000\u02cc\u02cd\u0001\u0000\u0000\u0000\u02cd"+
+    "\u02cb\u0001\u0000\u0000\u0000\u02cd\u02ce\u0001\u0000\u0000\u0000\u02ce"+
+    "\u02d6\u0001\u0000\u0000\u0000\u02cf\u02d3\u0003g,\u0000\u02d0\u02d2\u0003"+
+    "?\u0018\u0000\u02d1\u02d0\u0001\u0000\u0000\u0000\u02d2\u02d5\u0001\u0000"+
+    "\u0000\u0000\u02d3\u02d1\u0001\u0000\u0000\u0000\u02d3\u02d4\u0001\u0000"+
+    "\u0000\u0000\u02d4\u02d7\u0001\u0000\u0000\u0000\u02d5\u02d3\u0001\u0000"+
+    "\u0000\u0000\u02d6\u02cf\u0001\u0000\u0000\u0000\u02d6\u02d7\u0001\u0000"+
+    "\u0000\u0000\u02d7\u02d8\u0001\u0000\u0000\u0000\u02d8\u02d9\u0003G\u001c"+
+    "\u0000\u02d9\u02e3\u0001\u0000\u0000\u0000\u02da\u02dc\u0003g,\u0000\u02db"+
+    "\u02dd\u0003?\u0018\u0000\u02dc\u02db\u0001\u0000\u0000\u0000\u02dd\u02de"+
+    "\u0001\u0000\u0000\u0000\u02de\u02dc\u0001\u0000\u0000\u0000\u02de\u02df"+
+    "\u0001\u0000\u0000\u0000\u02df\u02e0\u0001\u0000\u0000\u0000\u02e0\u02e1"+
+    "\u0003G\u001c\u0000\u02e1\u02e3\u0001\u0000\u0000\u0000\u02e2\u02b9\u0001"+
+    "\u0000\u0000\u0000\u02e2\u02c4\u0001\u0000\u0000\u0000\u02e2\u02cb\u0001"+
+    "\u0000\u0000\u0000\u02e2\u02da\u0001\u0000\u0000\u0000\u02e3X\u0001\u0000"+
+    "\u0000\u0000\u02e4\u02e5\u0007\u001e\u0000\u0000\u02e5\u02e6\u0007\u001f"+
+    "\u0000\u0000\u02e6Z\u0001\u0000\u0000\u0000\u02e7\u02e8\u0007\f\u0000"+
+    "\u0000\u02e8\u02e9\u0007\t\u0000\u0000\u02e9\u02ea\u0007\u0000\u0000\u0000"+
+    "\u02ea\\\u0001\u0000\u0000\u0000\u02eb\u02ec\u0007\f\u0000\u0000\u02ec"+
+    "\u02ed\u0007\u0002\u0000\u0000\u02ed\u02ee\u0007\u0004\u0000\u0000\u02ee"+
+    "^\u0001\u0000\u0000\u0000\u02ef\u02f0\u0005=\u0000\u0000\u02f0`\u0001"+
+    "\u0000\u0000\u0000\u02f1\u02f2\u0005:\u0000\u0000\u02f2\u02f3\u0005:\u0000"+
+    "\u0000\u02f3b\u0001\u0000\u0000\u0000\u02f4\u02f5\u0005,\u0000\u0000\u02f5"+
+    "d\u0001\u0000\u0000\u0000\u02f6\u02f7\u0007\u0000\u0000\u0000\u02f7\u02f8"+
+    "\u0007\u0003\u0000\u0000\u02f8\u02f9\u0007\u0002\u0000\u0000\u02f9\u02fa"+
+    "\u0007\u0004\u0000\u0000\u02faf\u0001\u0000\u0000\u0000\u02fb\u02fc\u0005"+
+    ".\u0000\u0000\u02fch\u0001\u0000\u0000\u0000\u02fd\u02fe\u0007\u000f\u0000"+
+    "\u0000\u02fe\u02ff\u0007\f\u0000\u0000\u02ff\u0300\u0007\r\u0000\u0000"+
+    "\u0300\u0301\u0007\u0002\u0000\u0000\u0301\u0302\u0007\u0003\u0000\u0000"+
+    "\u0302j\u0001\u0000\u0000\u0000\u0303\u0304\u0007\u000f\u0000\u0000\u0304"+
+    "\u0305\u0007\u0001\u0000\u0000\u0305\u0306\u0007\u0006\u0000\u0000\u0306"+
+    "\u0307\u0007\u0002\u0000\u0000\u0307\u0308\u0007\u0005\u0000\u0000\u0308"+
+    "l\u0001\u0000\u0000\u0000\u0309\u030a\u0007\u0001\u0000\u0000\u030a\u030b"+
+    "\u0007\t\u0000\u0000\u030bn\u0001\u0000\u0000\u0000\u030c\u030d\u0007"+
+    "\u0001\u0000\u0000\u030d\u030e\u0007\u0002\u0000\u0000\u030ep\u0001\u0000"+
+    "\u0000\u0000\u030f\u0310\u0007\r\u0000\u0000\u0310\u0311\u0007\f\u0000"+
+    "\u0000\u0311\u0312\u0007\u0002\u0000\u0000\u0312\u0313\u0007\u0005\u0000"+
+    "\u0000\u0313r\u0001\u0000\u0000\u0000\u0314\u0315\u0007\r\u0000\u0000"+
+    "\u0315\u0316\u0007\u0001\u0000\u0000\u0316\u0317\u0007\u0012\u0000\u0000"+
+    "\u0317\u0318\u0007\u0003\u0000\u0000\u0318t\u0001\u0000\u0000\u0000\u0319"+
+    "\u031a\u0005(\u0000\u0000\u031av\u0001\u0000\u0000\u0000\u031b\u031c\u0007"+
+    "\t\u0000\u0000\u031c\u031d\u0007\u0007\u0000\u0000\u031d\u031e\u0007\u0005"+
+    "\u0000\u0000\u031ex\u0001\u0000\u0000\u0000\u031f\u0320\u0007\t\u0000"+
+    "\u0000\u0320\u0321\u0007\u0014\u0000\u0000\u0321\u0322\u0007\r\u0000\u0000"+
+    "\u0322\u0323\u0007\r\u0000\u0000\u0323z\u0001\u0000\u0000\u0000\u0324"+
+    "\u0325\u0007\t\u0000\u0000\u0325\u0326\u0007\u0014\u0000\u0000\u0326\u0327"+
+    "\u0007\r\u0000\u0000\u0327\u0328\u0007\r\u0000\u0000\u0328\u0329\u0007"+
+    "\u0002\u0000\u0000\u0329|\u0001\u0000\u0000\u0000\u032a\u032b\u0007\u0007"+
+    "\u0000\u0000\u032b\u032c\u0007\u0006\u0000\u0000\u032c~\u0001\u0000\u0000"+
+    "\u0000\u032d\u032e\u0005?\u0000\u0000\u032e\u0080\u0001\u0000\u0000\u0000"+
+    "\u032f\u0330\u0007\u0006\u0000\u0000\u0330\u0331\u0007\r\u0000\u0000\u0331"+
+    "\u0332\u0007\u0001\u0000\u0000\u0332\u0333\u0007\u0012\u0000\u0000\u0333"+
+    "\u0334\u0007\u0003\u0000\u0000\u0334\u0082\u0001\u0000\u0000\u0000\u0335"+
+    "\u0336\u0005)\u0000\u0000\u0336\u0084\u0001\u0000\u0000\u0000\u0337\u0338"+
+    "\u0007\u0005\u0000\u0000\u0338\u0339\u0007\u0006\u0000\u0000\u0339\u033a"+
+    "\u0007\u0014\u0000\u0000\u033a\u033b\u0007\u0003\u0000\u0000\u033b\u0086"+
+    "\u0001\u0000\u0000\u0000\u033c\u033d\u0005=\u0000\u0000\u033d\u033e\u0005"+
+    "=\u0000\u0000\u033e\u0088\u0001\u0000\u0000\u0000\u033f\u0340\u0005=\u0000"+
+    "\u0000\u0340\u0341\u0005~\u0000\u0000\u0341\u008a\u0001\u0000\u0000\u0000"+
+    "\u0342\u0343\u0005!\u0000\u0000\u0343\u0344\u0005=\u0000\u0000\u0344\u008c"+
+    "\u0001\u0000\u0000\u0000\u0345\u0346\u0005<\u0000\u0000\u0346\u008e\u0001"+
+    "\u0000\u0000\u0000\u0347\u0348\u0005<\u0000\u0000\u0348\u0349\u0005=\u0000"+
+    "\u0000\u0349\u0090\u0001\u0000\u0000\u0000\u034a\u034b\u0005>\u0000\u0000"+
+    "\u034b\u0092\u0001\u0000\u0000\u0000\u034c\u034d\u0005>\u0000\u0000\u034d"+
+    "\u034e\u0005=\u0000\u0000\u034e\u0094\u0001\u0000\u0000\u0000\u034f\u0350"+
+    "\u0005+\u0000\u0000\u0350\u0096\u0001\u0000\u0000\u0000\u0351\u0352\u0005"+
+    "-\u0000\u0000\u0352\u0098\u0001\u0000\u0000\u0000\u0353\u0354\u0005*\u0000"+
+    "\u0000\u0354\u009a\u0001\u0000\u0000\u0000\u0355\u0356\u0005/\u0000\u0000"+
+    "\u0356\u009c\u0001\u0000\u0000\u0000\u0357\u0358\u0005%\u0000\u0000\u0358"+
+    "\u009e\u0001\u0000\u0000\u0000\u0359\u035a\u0007\u0010\u0000\u0000\u035a"+
+    "\u035b\u0007\f\u0000\u0000\u035b\u035c\u0007\u0005\u0000\u0000\u035c\u035d"+
+    "\u0007\u0004\u0000\u0000\u035d\u035e\u0007\n\u0000\u0000\u035e\u00a0\u0001"+
+    "\u0000\u0000\u0000\u035f\u0360\u0003-\u000f\u0000\u0360\u0361\u0001\u0000"+
+    "\u0000\u0000\u0361\u0362\u0006I\f\u0000\u0362\u00a2\u0001\u0000\u0000"+
+    "\u0000\u0363\u0366\u0003\u007f8\u0000\u0364\u0367\u0003A\u0019\u0000\u0365"+
+    "\u0367\u0003O \u0000\u0366\u0364\u0001\u0000\u0000\u0000\u0366\u0365\u0001"+
+    "\u0000\u0000\u0000\u0367\u036b\u0001\u0000\u0000\u0000\u0368\u036a\u0003"+
+    "Q!\u0000\u0369\u0368\u0001\u0000\u0000\u0000\u036a\u036d\u0001\u0000\u0000"+
+    "\u0000\u036b\u0369\u0001\u0000\u0000\u0000\u036b\u036c\u0001\u0000\u0000"+
+    "\u0000\u036c\u0375\u0001\u0000\u0000\u0000\u036d\u036b\u0001\u0000\u0000"+
+    "\u0000\u036e\u0370\u0003\u007f8\u0000\u036f\u0371\u0003?\u0018\u0000\u0370"+
+    "\u036f\u0001\u0000\u0000\u0000\u0371\u0372\u0001\u0000\u0000\u0000\u0372"+
+    "\u0370\u0001\u0000\u0000\u0000\u0372\u0373\u0001\u0000\u0000\u0000\u0373"+
+    "\u0375\u0001\u0000\u0000\u0000\u0374\u0363\u0001\u0000\u0000\u0000\u0374"+
+    "\u036e\u0001\u0000\u0000\u0000\u0375\u00a4\u0001\u0000\u0000\u0000\u0376"+
+    "\u0377\u0005[\u0000\u0000\u0377\u0378\u0001\u0000\u0000\u0000\u0378\u0379"+
+    "\u0006K\u0000\u0000\u0379\u037a\u0006K\u0000\u0000\u037a\u00a6\u0001\u0000"+
+    "\u0000\u0000\u037b\u037c\u0005]\u0000\u0000\u037c\u037d\u0001\u0000\u0000"+
+    "\u0000\u037d\u037e\u0006L\u000b\u0000\u037e\u037f\u0006L\u000b\u0000\u037f"+
+    "\u00a8\u0001\u0000\u0000\u0000\u0380\u0384\u0003A\u0019\u0000\u0381\u0383"+
+    "\u0003Q!\u0000\u0382\u0381\u0001\u0000\u0000\u0000\u0383\u0386\u0001\u0000"+
+    "\u0000\u0000\u0384\u0382\u0001\u0000\u0000\u0000\u0384\u0385\u0001\u0000"+
+    "\u0000\u0000\u0385\u0391\u0001\u0000\u0000\u0000\u0386\u0384\u0001\u0000"+
+    "\u0000\u0000\u0387\u038a\u0003O \u0000\u0388\u038a\u0003I\u001d\u0000"+
+    "\u0389\u0387\u0001\u0000\u0000\u0000\u0389\u0388\u0001\u0000\u0000\u0000"+
+    "\u038a\u038c\u0001\u0000\u0000\u0000\u038b\u038d\u0003Q!\u0000\u038c\u038b"+
+    "\u0001\u0000\u0000\u0000\u038d\u038e\u0001\u0000\u0000\u0000\u038e\u038c"+
+    "\u0001\u0000\u0000\u0000\u038e\u038f\u0001\u0000\u0000\u0000\u038f\u0391"+
+    "\u0001\u0000\u0000\u0000\u0390\u0380\u0001\u0000\u0000\u0000\u0390\u0389"+
+    "\u0001\u0000\u0000\u0000\u0391\u00aa\u0001\u0000\u0000\u0000\u0392\u0394"+
+    "\u0003K\u001e\u0000\u0393\u0395\u0003M\u001f\u0000\u0394\u0393\u0001\u0000"+
+    "\u0000\u0000\u0395\u0396\u0001\u0000\u0000\u0000\u0396\u0394\u0001\u0000"+
+    "\u0000\u0000\u0396\u0397\u0001\u0000\u0000\u0000\u0397\u0398\u0001\u0000"+
+    "\u0000\u0000\u0398\u0399\u0003K\u001e\u0000\u0399\u00ac\u0001\u0000\u0000"+
+    "\u0000\u039a\u039b\u0003\u00abN\u0000\u039b\u00ae\u0001\u0000\u0000\u0000"+
+    "\u039c\u039d\u00037\u0014\u0000\u039d\u039e\u0001\u0000\u0000\u0000\u039e"+
+    "\u039f\u0006P\n\u0000\u039f\u00b0\u0001\u0000\u0000\u0000\u03a0\u03a1"+
+    "\u00039\u0015\u0000\u03a1\u03a2\u0001\u0000\u0000\u0000\u03a2\u03a3\u0006"+
+    "Q\n\u0000\u03a3\u00b2\u0001\u0000\u0000\u0000\u03a4\u03a5\u0003;\u0016"+
+    "\u0000\u03a5\u03a6\u0001\u0000\u0000\u0000\u03a6\u03a7\u0006R\n\u0000"+
+    "\u03a7\u00b4\u0001\u0000\u0000\u0000\u03a8\u03a9\u0003\u00a5K\u0000\u03a9"+
+    "\u03aa\u0001\u0000\u0000\u0000\u03aa\u03ab\u0006S\r\u0000\u03ab\u03ac"+
+    "\u0006S\u000e\u0000\u03ac\u00b6\u0001\u0000\u0000\u0000\u03ad\u03ae\u0003"+
+    "=\u0017\u0000\u03ae\u03af\u0001\u0000\u0000\u0000\u03af\u03b0\u0006T\u000f"+
+    "\u0000\u03b0\u03b1\u0006T\u000b\u0000\u03b1\u00b8\u0001\u0000\u0000\u0000"+
+    "\u03b2\u03b3\u0003;\u0016\u0000\u03b3\u03b4\u0001\u0000\u0000\u0000\u03b4"+
+    "\u03b5\u0006U\n\u0000\u03b5\u00ba\u0001\u0000\u0000\u0000\u03b6\u03b7"+
+    "\u00037\u0014\u0000\u03b7\u03b8\u0001\u0000\u0000\u0000\u03b8\u03b9\u0006"+
+    "V\n\u0000\u03b9\u00bc\u0001\u0000\u0000\u0000\u03ba\u03bb\u00039\u0015"+
+    "\u0000\u03bb\u03bc\u0001\u0000\u0000\u0000\u03bc\u03bd\u0006W\n\u0000"+
+    "\u03bd\u00be\u0001\u0000\u0000\u0000\u03be\u03bf\u0003=\u0017\u0000\u03bf"+
+    "\u03c0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006X\u000f\u0000\u03c1\u03c2"+
+    "\u0006X\u000b\u0000\u03c2\u00c0\u0001\u0000\u0000\u0000\u03c3\u03c4\u0003"+
+    "\u00a5K\u0000\u03c4\u03c5\u0001\u0000\u0000\u0000\u03c5\u03c6\u0006Y\r"+
+    "\u0000\u03c6\u00c2\u0001\u0000\u0000\u0000\u03c7\u03c8\u0003\u00a7L\u0000"+
+    "\u03c8\u03c9\u0001\u0000\u0000\u0000\u03c9\u03ca\u0006Z\u0010\u0000\u03ca"+
+    "\u00c4\u0001\u0000\u0000\u0000\u03cb\u03cc\u0003\u0151\u00a1\u0000\u03cc"+
+    "\u03cd\u0001\u0000\u0000\u0000\u03cd\u03ce\u0006[\u0011\u0000\u03ce\u00c6"+
+    "\u0001\u0000\u0000\u0000\u03cf\u03d0\u0003c*\u0000\u03d0\u03d1\u0001\u0000"+
+    "\u0000\u0000\u03d1\u03d2\u0006\\\u0012\u0000\u03d2\u00c8\u0001\u0000\u0000"+
+    "\u0000\u03d3\u03d4\u0003_(\u0000\u03d4\u03d5\u0001\u0000\u0000\u0000\u03d5"+
+    "\u03d6\u0006]\u0013\u0000\u03d6\u00ca\u0001\u0000\u0000\u0000\u03d7\u03d8"+
+    "\u0007\u0010\u0000\u0000\u03d8\u03d9\u0007\u0003\u0000\u0000\u03d9\u03da"+
+    "\u0007\u0005\u0000\u0000\u03da\u03db\u0007\f\u0000\u0000\u03db\u03dc\u0007"+
+    "\u0000\u0000\u0000\u03dc\u03dd\u0007\f\u0000\u0000\u03dd\u03de\u0007\u0005"+
+    "\u0000\u0000\u03de\u03df\u0007\f\u0000\u0000\u03df\u00cc\u0001\u0000\u0000"+
+    "\u0000\u03e0\u03e4\b \u0000\u0000\u03e1\u03e2\u0005/\u0000\u0000\u03e2"+
+    "\u03e4\b!\u0000\u0000\u03e3\u03e0\u0001\u0000\u0000\u0000\u03e3\u03e1"+
+    "\u0001\u0000\u0000\u0000\u03e4\u00ce\u0001\u0000\u0000\u0000\u03e5\u03e7"+
+    "\u0003\u00cd_\u0000\u03e6\u03e5\u0001\u0000\u0000\u0000\u03e7\u03e8\u0001"+
+    "\u0000\u0000\u0000\u03e8\u03e6\u0001\u0000\u0000\u0000\u03e8\u03e9\u0001"+
+    "\u0000\u0000\u0000\u03e9\u00d0\u0001\u0000\u0000\u0000\u03ea\u03eb\u0003"+
+    "\u00cf`\u0000\u03eb\u03ec\u0001\u0000\u0000\u0000\u03ec\u03ed\u0006a\u0014"+
+    "\u0000\u03ed\u00d2\u0001\u0000\u0000\u0000\u03ee\u03ef\u0003S\"\u0000"+
+    "\u03ef\u03f0\u0001\u0000\u0000\u0000\u03f0\u03f1\u0006b\u0015\u0000\u03f1"+
+    "\u00d4\u0001\u0000\u0000\u0000\u03f2\u03f3\u00037\u0014\u0000\u03f3\u03f4"+
+    "\u0001\u0000\u0000\u0000\u03f4\u03f5\u0006c\n\u0000\u03f5\u00d6\u0001"+
+    "\u0000\u0000\u0000\u03f6\u03f7\u00039\u0015\u0000\u03f7\u03f8\u0001\u0000"+
+    "\u0000\u0000\u03f8\u03f9\u0006d\n\u0000\u03f9\u00d8\u0001\u0000\u0000"+
+    "\u0000\u03fa\u03fb\u0003;\u0016\u0000\u03fb\u03fc\u0001\u0000\u0000\u0000"+
+    "\u03fc\u03fd\u0006e\n\u0000\u03fd\u00da\u0001\u0000\u0000\u0000\u03fe"+
+    "\u03ff\u0003=\u0017\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400\u0401"+
+    "\u0006f\u000f\u0000\u0401\u0402\u0006f\u000b\u0000\u0402\u00dc\u0001\u0000"+
+    "\u0000\u0000\u0403\u0404\u0003g,\u0000\u0404\u0405\u0001\u0000\u0000\u0000"+
+    "\u0405\u0406\u0006g\u0016\u0000\u0406\u00de\u0001\u0000\u0000\u0000\u0407"+
+    "\u0408\u0003c*\u0000\u0408\u0409\u0001\u0000\u0000\u0000\u0409\u040a\u0006"+
+    "h\u0012\u0000\u040a\u00e0\u0001\u0000\u0000\u0000\u040b\u040c\u0004i\u0003"+
+    "\u0000\u040c\u040d\u0003\u007f8\u0000\u040d\u040e\u0001\u0000\u0000\u0000"+
+    "\u040e\u040f\u0006i\u0017\u0000\u040f\u00e2\u0001\u0000\u0000\u0000\u0410"+
+    "\u0411\u0004j\u0004\u0000\u0411\u0412\u0003\u00a3J\u0000\u0412\u0413\u0001"+
+    "\u0000\u0000\u0000\u0413\u0414\u0006j\u0018\u0000\u0414\u00e4\u0001\u0000"+
+    "\u0000\u0000\u0415\u041a\u0003A\u0019\u0000\u0416\u041a\u0003?\u0018\u0000"+
+    "\u0417\u041a\u0003O \u0000\u0418\u041a\u0003\u0099E\u0000\u0419\u0415"+
+    "\u0001\u0000\u0000\u0000\u0419\u0416\u0001\u0000\u0000\u0000\u0419\u0417"+
+    "\u0001\u0000\u0000\u0000\u0419\u0418\u0001\u0000\u0000\u0000\u041a\u00e6"+
+    "\u0001\u0000\u0000\u0000\u041b\u041e\u0003A\u0019\u0000\u041c\u041e\u0003"+
+    "\u0099E\u0000\u041d\u041b\u0001\u0000\u0000\u0000\u041d\u041c\u0001\u0000"+
+    "\u0000\u0000\u041e\u0422\u0001\u0000\u0000\u0000\u041f\u0421\u0003\u00e5"+
+    "k\u0000\u0420\u041f\u0001\u0000\u0000\u0000\u0421\u0424\u0001\u0000\u0000"+
+    "\u0000\u0422\u0420\u0001\u0000\u0000\u0000\u0422\u0423\u0001\u0000\u0000"+
+    "\u0000\u0423\u042f\u0001\u0000\u0000\u0000\u0424\u0422\u0001\u0000\u0000"+
+    "\u0000\u0425\u0428\u0003O \u0000\u0426\u0428\u0003I\u001d\u0000\u0427"+
+    "\u0425\u0001\u0000\u0000\u0000\u0427\u0426\u0001\u0000\u0000\u0000\u0428"+
+    "\u042a\u0001\u0000\u0000\u0000\u0429\u042b\u0003\u00e5k\u0000\u042a\u0429"+
+    "\u0001\u0000\u0000\u0000\u042b\u042c\u0001\u0000\u0000\u0000\u042c\u042a"+
+    "\u0001\u0000\u0000\u0000\u042c\u042d\u0001\u0000\u0000\u0000\u042d\u042f"+
+    "\u0001\u0000\u0000\u0000\u042e\u041d\u0001\u0000\u0000\u0000\u042e\u0427"+
+    "\u0001\u0000\u0000\u0000\u042f\u00e8\u0001\u0000\u0000\u0000\u0430\u0433"+
+    "\u0003\u00e7l\u0000\u0431\u0433\u0003\u00abN\u0000\u0432\u0430\u0001\u0000"+
+    "\u0000\u0000\u0432\u0431\u0001\u0000\u0000\u0000\u0433\u0434\u0001\u0000"+
+    "\u0000\u0000\u0434\u0432\u0001\u0000\u0000\u0000\u0434\u0435\u0001\u0000"+
+    "\u0000\u0000\u0435\u00ea\u0001\u0000\u0000\u0000\u0436\u0437\u00037\u0014"+
+    "\u0000\u0437\u0438\u0001\u0000\u0000\u0000\u0438\u0439\u0006n\n\u0000"+
+    "\u0439\u00ec\u0001\u0000\u0000\u0000\u043a\u043b\u00039\u0015\u0000\u043b"+
+    "\u043c\u0001\u0000\u0000\u0000\u043c\u043d\u0006o\n\u0000\u043d\u00ee"+
+    "\u0001\u0000\u0000\u0000\u043e\u043f\u0003;\u0016\u0000\u043f\u0440\u0001"+
+    "\u0000\u0000\u0000\u0440\u0441\u0006p\n\u0000\u0441\u00f0\u0001\u0000"+
+    "\u0000\u0000\u0442\u0443\u0003=\u0017\u0000\u0443\u0444\u0001\u0000\u0000"+
+    "\u0000\u0444\u0445\u0006q\u000f\u0000\u0445\u0446\u0006q\u000b\u0000\u0446"+
+    "\u00f2\u0001\u0000\u0000\u0000\u0447\u0448\u0003_(\u0000\u0448\u0449\u0001"+
+    "\u0000\u0000\u0000\u0449\u044a\u0006r\u0013\u0000\u044a\u00f4\u0001\u0000"+
+    "\u0000\u0000\u044b\u044c\u0003c*\u0000\u044c\u044d\u0001\u0000\u0000\u0000"+
+    "\u044d\u044e\u0006s\u0012\u0000\u044e\u00f6\u0001\u0000\u0000\u0000\u044f"+
+    "\u0450\u0003g,\u0000\u0450\u0451\u0001\u0000\u0000\u0000\u0451\u0452\u0006"+
+    "t\u0016\u0000\u0452\u00f8\u0001\u0000\u0000\u0000\u0453\u0454\u0004u\u0005"+
+    "\u0000\u0454\u0455\u0003\u007f8\u0000\u0455\u0456\u0001\u0000\u0000\u0000"+
+    "\u0456\u0457\u0006u\u0017\u0000\u0457\u00fa\u0001\u0000\u0000\u0000\u0458"+
+    "\u0459\u0004v\u0006\u0000\u0459\u045a\u0003\u00a3J\u0000\u045a\u045b\u0001"+
+    "\u0000\u0000\u0000\u045b\u045c\u0006v\u0018\u0000\u045c\u00fc\u0001\u0000"+
+    "\u0000\u0000\u045d\u045e\u0007\f\u0000\u0000\u045e\u045f\u0007\u0002\u0000"+
+    "\u0000\u045f\u00fe\u0001\u0000\u0000\u0000\u0460\u0461\u0003\u00e9m\u0000"+
+    "\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463\u0006x\u0019\u0000\u0463"+
+    "\u0100\u0001\u0000\u0000\u0000\u0464\u0465\u00037\u0014\u0000\u0465\u0466"+
+    "\u0001\u0000\u0000\u0000\u0466\u0467\u0006y\n\u0000\u0467\u0102\u0001"+
+    "\u0000\u0000\u0000\u0468\u0469\u00039\u0015\u0000\u0469\u046a\u0001\u0000"+
+    "\u0000\u0000\u046a\u046b\u0006z\n\u0000\u046b\u0104\u0001\u0000\u0000"+
+    "\u0000\u046c\u046d\u0003;\u0016\u0000\u046d\u046e\u0001\u0000\u0000\u0000"+
+    "\u046e\u046f\u0006{\n\u0000\u046f\u0106\u0001\u0000\u0000\u0000\u0470"+
+    "\u0471\u0003=\u0017\u0000\u0471\u0472\u0001\u0000\u0000\u0000\u0472\u0473"+
+    "\u0006|\u000f\u0000\u0473\u0474\u0006|\u000b\u0000\u0474\u0108\u0001\u0000"+
+    "\u0000\u0000\u0475\u0476\u0003\u00a5K\u0000\u0476\u0477\u0001\u0000\u0000"+
+    "\u0000\u0477\u0478\u0006}\r\u0000\u0478\u0479\u0006}\u001a\u0000\u0479"+
+    "\u010a\u0001\u0000\u0000\u0000\u047a\u047b\u0007\u0007\u0000\u0000\u047b"+
+    "\u047c\u0007\t\u0000\u0000\u047c\u047d\u0001\u0000\u0000\u0000\u047d\u047e"+
+    "\u0006~\u001b\u0000\u047e\u010c\u0001\u0000\u0000\u0000\u047f\u0480\u0007"+
+    "\u0013\u0000\u0000\u0480\u0481\u0007\u0001\u0000\u0000\u0481\u0482\u0007"+
+    "\u0005\u0000\u0000\u0482\u0483\u0007\n\u0000\u0000\u0483\u0484\u0001\u0000"+
+    "\u0000\u0000\u0484\u0485\u0006\u007f\u001b\u0000\u0485\u010e\u0001\u0000"+
+    "\u0000\u0000\u0486\u0487\b\"\u0000\u0000\u0487\u0110\u0001\u0000\u0000"+
+    "\u0000\u0488\u048a\u0003\u010f\u0080\u0000\u0489\u0488\u0001\u0000\u0000"+
+    "\u0000\u048a\u048b\u0001\u0000\u0000\u0000\u048b\u0489\u0001\u0000\u0000"+
+    "\u0000\u048b\u048c\u0001\u0000\u0000\u0000\u048c\u048d\u0001\u0000\u0000"+
+    "\u0000\u048d\u048e\u0003\u0151\u00a1\u0000\u048e\u0490\u0001\u0000\u0000"+
+    "\u0000\u048f\u0489\u0001\u0000\u0000\u0000\u048f\u0490\u0001\u0000\u0000"+
+    "\u0000\u0490\u0492\u0001\u0000\u0000\u0000\u0491\u0493\u0003\u010f\u0080"+
+    "\u0000\u0492\u0491\u0001\u0000\u0000\u0000\u0493\u0494\u0001\u0000\u0000"+
+    "\u0000\u0494\u0492\u0001\u0000\u0000\u0000\u0494\u0495\u0001\u0000\u0000"+
+    "\u0000\u0495\u0112\u0001\u0000\u0000\u0000\u0496\u0497\u0003\u0111\u0081"+
+    "\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498\u0499\u0006\u0082\u001c"+
+    "\u0000\u0499\u0114\u0001\u0000\u0000\u0000\u049a\u049b\u00037\u0014\u0000"+
+    "\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d\u0006\u0083\n\u0000\u049d"+
+    "\u0116\u0001\u0000\u0000\u0000\u049e\u049f\u00039\u0015\u0000\u049f\u04a0"+
+    "\u0001\u0000\u0000\u0000\u04a0\u04a1\u0006\u0084\n\u0000\u04a1\u0118\u0001"+
+    "\u0000\u0000\u0000\u04a2\u04a3\u0003;\u0016\u0000\u04a3\u04a4\u0001\u0000"+
+    "\u0000\u0000\u04a4\u04a5\u0006\u0085\n\u0000\u04a5\u011a\u0001\u0000\u0000"+
+    "\u0000\u04a6\u04a7\u0003=\u0017\u0000\u04a7\u04a8\u0001\u0000\u0000\u0000"+
+    "\u04a8\u04a9\u0006\u0086\u000f\u0000\u04a9\u04aa\u0006\u0086\u000b\u0000"+
+    "\u04aa\u04ab\u0006\u0086\u000b\u0000\u04ab\u011c\u0001\u0000\u0000\u0000"+
+    "\u04ac\u04ad\u0003_(\u0000\u04ad\u04ae\u0001\u0000\u0000\u0000\u04ae\u04af"+
+    "\u0006\u0087\u0013\u0000\u04af\u011e\u0001\u0000\u0000\u0000\u04b0\u04b1"+
+    "\u0003c*\u0000\u04b1\u04b2\u0001\u0000\u0000\u0000\u04b2\u04b3\u0006\u0088"+
+    "\u0012\u0000\u04b3\u0120\u0001\u0000\u0000\u0000\u04b4\u04b5\u0003g,\u0000"+
+    "\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6\u04b7\u0006\u0089\u0016\u0000"+
+    "\u04b7\u0122\u0001\u0000\u0000\u0000\u04b8\u04b9\u0003\u010d\u007f\u0000"+
+    "\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba\u04bb\u0006\u008a\u001d\u0000"+
+    "\u04bb\u0124\u0001\u0000\u0000\u0000\u04bc\u04bd\u0003\u00e9m\u0000\u04bd"+
+    "\u04be\u0001\u0000\u0000\u0000\u04be\u04bf\u0006\u008b\u0019\u0000\u04bf"+
+    "\u0126\u0001\u0000\u0000\u0000\u04c0\u04c1\u0003\u00adO\u0000\u04c1\u04c2"+
+    "\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006\u008c\u001e\u0000\u04c3\u0128"+
+    "\u0001\u0000\u0000\u0000\u04c4\u04c5\u0004\u008d\u0007\u0000\u04c5\u04c6"+
+    "\u0003\u007f8\u0000\u04c6\u04c7\u0001\u0000\u0000\u0000\u04c7\u04c8\u0006"+
+    "\u008d\u0017\u0000\u04c8\u012a\u0001\u0000\u0000\u0000\u04c9\u04ca\u0004"+
+    "\u008e\b\u0000\u04ca\u04cb\u0003\u00a3J\u0000\u04cb\u04cc\u0001\u0000"+
+    "\u0000\u0000\u04cc\u04cd\u0006\u008e\u0018\u0000\u04cd\u012c\u0001\u0000"+
+    "\u0000\u0000\u04ce\u04cf\u00037\u0014\u0000\u04cf\u04d0\u0001\u0000\u0000"+
+    "\u0000\u04d0\u04d1\u0006\u008f\n\u0000\u04d1\u012e\u0001\u0000\u0000\u0000"+
+    "\u04d2\u04d3\u00039\u0015\u0000\u04d3\u04d4\u0001\u0000\u0000\u0000\u04d4"+
+    "\u04d5\u0006\u0090\n\u0000\u04d5\u0130\u0001\u0000\u0000\u0000\u04d6\u04d7"+
+    "\u0003;\u0016\u0000\u04d7\u04d8\u0001\u0000\u0000\u0000\u04d8\u04d9\u0006"+
+    "\u0091\n\u0000\u04d9\u0132\u0001\u0000\u0000\u0000\u04da\u04db\u0003="+
+    "\u0017\u0000\u04db\u04dc\u0001\u0000\u0000\u0000\u04dc\u04dd\u0006\u0092"+
+    "\u000f\u0000\u04dd\u04de\u0006\u0092\u000b\u0000\u04de\u0134\u0001\u0000"+
+    "\u0000\u0000\u04df\u04e0\u0003g,\u0000\u04e0\u04e1\u0001\u0000\u0000\u0000"+
+    "\u04e1\u04e2\u0006\u0093\u0016\u0000\u04e2\u0136\u0001\u0000\u0000\u0000"+
+    "\u04e3\u04e4\u0004\u0094\t\u0000\u04e4\u04e5\u0003\u007f8\u0000\u04e5"+
+    "\u04e6\u0001\u0000\u0000\u0000\u04e6\u04e7\u0006\u0094\u0017\u0000\u04e7"+
+    "\u0138\u0001\u0000\u0000\u0000\u04e8\u04e9\u0004\u0095\n\u0000\u04e9\u04ea"+
+    "\u0003\u00a3J\u0000\u04ea\u04eb\u0001\u0000\u0000\u0000\u04eb\u04ec\u0006"+
+    "\u0095\u0018\u0000\u04ec\u013a\u0001\u0000\u0000\u0000\u04ed\u04ee\u0003"+
+    "\u00adO\u0000\u04ee\u04ef\u0001\u0000\u0000\u0000\u04ef\u04f0\u0006\u0096"+
+    "\u001e\u0000\u04f0\u013c\u0001\u0000\u0000\u0000\u04f1\u04f2\u0003\u00a9"+
+    "M\u0000\u04f2\u04f3\u0001\u0000\u0000\u0000\u04f3\u04f4\u0006\u0097\u001f"+
+    "\u0000\u04f4\u013e\u0001\u0000\u0000\u0000\u04f5\u04f6\u00037\u0014\u0000"+
+    "\u04f6\u04f7\u0001\u0000\u0000\u0000\u04f7\u04f8\u0006\u0098\n\u0000\u04f8"+
+    "\u0140\u0001\u0000\u0000\u0000\u04f9\u04fa\u00039\u0015\u0000\u04fa\u04fb"+
+    "\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u0099\n\u0000\u04fc\u0142\u0001"+
+    "\u0000\u0000\u0000\u04fd\u04fe\u0003;\u0016\u0000\u04fe\u04ff\u0001\u0000"+
+    "\u0000\u0000\u04ff\u0500\u0006\u009a\n\u0000\u0500\u0144\u0001\u0000\u0000"+
+    "\u0000\u0501\u0502\u0003=\u0017\u0000\u0502\u0503\u0001\u0000\u0000\u0000"+
+    "\u0503\u0504\u0006\u009b\u000f\u0000\u0504\u0505\u0006\u009b\u000b\u0000"+
+    "\u0505\u0146\u0001\u0000\u0000\u0000\u0506\u0507\u0007\u0001\u0000\u0000"+
+    "\u0507\u0508\u0007\t\u0000\u0000\u0508\u0509\u0007\u000f\u0000\u0000\u0509"+
+    "\u050a\u0007\u0007\u0000\u0000\u050a\u0148\u0001\u0000\u0000\u0000\u050b"+
+    "\u050c\u00037\u0014\u0000\u050c\u050d\u0001\u0000\u0000\u0000\u050d\u050e"+
+    "\u0006\u009d\n\u0000\u050e\u014a\u0001\u0000\u0000\u0000\u050f\u0510\u0003"+
+    "9\u0015\u0000\u0510\u0511\u0001\u0000\u0000\u0000\u0511\u0512\u0006\u009e"+
+    "\n\u0000\u0512\u014c\u0001\u0000\u0000\u0000\u0513\u0514\u0003;\u0016"+
+    "\u0000\u0514\u0515\u0001\u0000\u0000\u0000\u0515\u0516\u0006\u009f\n\u0000"+
+    "\u0516\u014e\u0001\u0000\u0000\u0000\u0517\u0518\u0003\u00a7L\u0000\u0518"+
+    "\u0519\u0001\u0000\u0000\u0000\u0519\u051a\u0006\u00a0\u0010\u0000\u051a"+
+    "\u051b\u0006\u00a0\u000b\u0000\u051b\u0150\u0001\u0000\u0000\u0000\u051c"+
+    "\u051d\u0005:\u0000\u0000\u051d\u0152\u0001\u0000\u0000\u0000\u051e\u0524"+
+    "\u0003I\u001d\u0000\u051f\u0524\u0003?\u0018\u0000\u0520\u0524\u0003g"+
+    ",\u0000\u0521\u0524\u0003A\u0019\u0000\u0522\u0524\u0003O \u0000\u0523"+
+    "\u051e\u0001\u0000\u0000\u0000\u0523\u051f\u0001\u0000\u0000\u0000\u0523"+
+    "\u0520\u0001\u0000\u0000\u0000\u0523\u0521\u0001\u0000\u0000\u0000\u0523"+
+    "\u0522\u0001\u0000\u0000\u0000\u0524\u0525\u0001\u0000\u0000\u0000\u0525"+
+    "\u0523\u0001\u0000\u0000\u0000\u0525\u0526\u0001\u0000\u0000\u0000\u0526"+
+    "\u0154\u0001\u0000\u0000\u0000\u0527\u0528\u00037\u0014\u0000\u0528\u0529"+
+    "\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u00a3\n\u0000\u052a\u0156\u0001"+
+    "\u0000\u0000\u0000\u052b\u052c\u00039\u0015\u0000\u052c\u052d\u0001\u0000"+
+    "\u0000\u0000\u052d\u052e\u0006\u00a4\n\u0000\u052e\u0158\u0001\u0000\u0000"+
+    "\u0000\u052f\u0530\u0003;\u0016\u0000\u0530\u0531\u0001\u0000\u0000\u0000"+
+    "\u0531\u0532\u0006\u00a5\n\u0000\u0532\u015a\u0001\u0000\u0000\u0000\u0533"+
+    "\u0534\u0003=\u0017\u0000\u0534\u0535\u0001\u0000\u0000\u0000\u0535\u0536"+
+    "\u0006\u00a6\u000f\u0000\u0536\u0537\u0006\u00a6\u000b\u0000\u0537\u015c"+
+    "\u0001\u0000\u0000\u0000\u0538\u0539\u0003\u0151\u00a1\u0000\u0539\u053a"+
+    "\u0001\u0000\u0000\u0000\u053a\u053b\u0006\u00a7\u0011\u0000\u053b\u015e"+
+    "\u0001\u0000\u0000\u0000\u053c\u053d\u0003c*\u0000\u053d\u053e\u0001\u0000"+
+    "\u0000\u0000\u053e\u053f\u0006\u00a8\u0012\u0000\u053f\u0160\u0001\u0000"+
+    "\u0000\u0000\u0540\u0541\u0003g,\u0000\u0541\u0542\u0001\u0000\u0000\u0000"+
+    "\u0542\u0543\u0006\u00a9\u0016\u0000\u0543\u0162\u0001\u0000\u0000\u0000"+
+    "\u0544\u0545\u0003\u010b~\u0000\u0545\u0546\u0001\u0000\u0000\u0000\u0546"+
+    "\u0547\u0006\u00aa \u0000\u0547\u0548\u0006\u00aa!\u0000\u0548\u0164\u0001"+
+    "\u0000\u0000\u0000\u0549\u054a\u0003\u00cf`\u0000\u054a\u054b\u0001\u0000"+
+    "\u0000\u0000\u054b\u054c\u0006\u00ab\u0014\u0000\u054c\u0166\u0001\u0000"+
+    "\u0000\u0000\u054d\u054e\u0003S\"\u0000\u054e\u054f\u0001\u0000\u0000"+
+    "\u0000\u054f\u0550\u0006\u00ac\u0015\u0000\u0550\u0168\u0001\u0000\u0000"+
+    "\u0000\u0551\u0552\u00037\u0014\u0000\u0552\u0553\u0001\u0000\u0000\u0000"+
+    "\u0553\u0554\u0006\u00ad\n\u0000\u0554\u016a\u0001\u0000\u0000\u0000\u0555"+
+    "\u0556\u00039\u0015\u0000\u0556\u0557\u0001\u0000\u0000\u0000\u0557\u0558"+
+    "\u0006\u00ae\n\u0000\u0558\u016c\u0001\u0000\u0000\u0000\u0559\u055a\u0003"+
+    ";\u0016\u0000\u055a\u055b\u0001\u0000\u0000\u0000\u055b\u055c\u0006\u00af"+
+    "\n\u0000\u055c\u016e\u0001\u0000\u0000\u0000\u055d\u055e\u0003=\u0017"+
+    "\u0000\u055e\u055f\u0001\u0000\u0000\u0000\u055f\u0560\u0006\u00b0\u000f"+
+    "\u0000\u0560\u0561\u0006\u00b0\u000b\u0000\u0561\u0562\u0006\u00b0\u000b"+
+    "\u0000\u0562\u0170\u0001\u0000\u0000\u0000\u0563\u0564\u0003c*\u0000\u0564"+
+    "\u0565\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00b1\u0012\u0000\u0566"+
+    "\u0172\u0001\u0000\u0000\u0000\u0567\u0568\u0003g,\u0000\u0568\u0569\u0001"+
+    "\u0000\u0000\u0000\u0569\u056a\u0006\u00b2\u0016\u0000\u056a\u0174\u0001"+
+    "\u0000\u0000\u0000\u056b\u056c\u0003\u00e9m\u0000\u056c\u056d\u0001\u0000"+
+    "\u0000\u0000\u056d\u056e\u0006\u00b3\u0019\u0000\u056e\u0176\u0001\u0000"+
+    "\u0000\u0000\u056f\u0570\u00037\u0014\u0000\u0570\u0571\u0001\u0000\u0000"+
+    "\u0000\u0571\u0572\u0006\u00b4\n\u0000\u0572\u0178\u0001\u0000\u0000\u0000"+
+    "\u0573\u0574\u00039\u0015\u0000\u0574\u0575\u0001\u0000\u0000\u0000\u0575"+
+    "\u0576\u0006\u00b5\n\u0000\u0576\u017a\u0001\u0000\u0000\u0000\u0577\u0578"+
+    "\u0003;\u0016\u0000\u0578\u0579\u0001\u0000\u0000\u0000\u0579\u057a\u0006"+
+    "\u00b6\n\u0000\u057a\u017c\u0001\u0000\u0000\u0000\u057b\u057c\u0003="+
+    "\u0017\u0000\u057c\u057d\u0001\u0000\u0000\u0000\u057d\u057e\u0006\u00b7"+
+    "\u000f\u0000\u057e\u057f\u0006\u00b7\u000b\u0000\u057f\u017e\u0001\u0000"+
+    "\u0000\u0000\u0580\u0581\u0003\u00cf`\u0000\u0581\u0582\u0001\u0000\u0000"+
+    "\u0000\u0582\u0583\u0006\u00b8\u0014\u0000\u0583\u0584\u0006\u00b8\u000b"+
+    "\u0000\u0584\u0585\u0006\u00b8\"\u0000\u0585\u0180\u0001\u0000\u0000\u0000"+
+    "\u0586\u0587\u0003S\"\u0000\u0587\u0588\u0001\u0000\u0000\u0000\u0588"+
+    "\u0589\u0006\u00b9\u0015\u0000\u0589\u058a\u0006\u00b9\u000b\u0000\u058a"+
+    "\u058b\u0006\u00b9\"\u0000\u058b\u0182\u0001\u0000\u0000\u0000\u058c\u058d"+
+    "\u00037\u0014\u0000\u058d\u058e\u0001\u0000\u0000\u0000\u058e\u058f\u0006"+
+    "\u00ba\n\u0000\u058f\u0184\u0001\u0000\u0000\u0000\u0590\u0591\u00039"+
+    "\u0015\u0000\u0591\u0592\u0001\u0000\u0000\u0000\u0592\u0593\u0006\u00bb"+
+    "\n\u0000\u0593\u0186\u0001\u0000\u0000\u0000\u0594\u0595\u0003;\u0016"+
+    "\u0000\u0595\u0596\u0001\u0000\u0000\u0000\u0596\u0597\u0006\u00bc\n\u0000"+
+    "\u0597\u0188\u0001\u0000\u0000\u0000\u0598\u0599\u0003\u0151\u00a1\u0000"+
+    "\u0599\u059a\u0001\u0000\u0000\u0000\u059a\u059b\u0006\u00bd\u0011\u0000"+
+    "\u059b\u059c\u0006\u00bd\u000b\u0000\u059c\u059d\u0006\u00bd\t\u0000\u059d"+
+    "\u018a\u0001\u0000\u0000\u0000\u059e\u059f\u0003c*\u0000\u059f\u05a0\u0001"+
+    "\u0000\u0000\u0000\u05a0\u05a1\u0006\u00be\u0012\u0000\u05a1\u05a2\u0006"+
+    "\u00be\u000b\u0000\u05a2\u05a3\u0006\u00be\t\u0000\u05a3\u018c\u0001\u0000"+
+    "\u0000\u0000\u05a4\u05a5\u00037\u0014\u0000\u05a5\u05a6\u0001\u0000\u0000"+
+    "\u0000\u05a6\u05a7\u0006\u00bf\n\u0000\u05a7\u018e\u0001\u0000\u0000\u0000"+
+    "\u05a8\u05a9\u00039\u0015\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa"+
+    "\u05ab\u0006\u00c0\n\u0000\u05ab\u0190\u0001\u0000\u0000\u0000\u05ac\u05ad"+
+    "\u0003;\u0016\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006"+
+    "\u00c1\n\u0000\u05af\u0192\u0001\u0000\u0000\u0000\u05b0\u05b1\u0003\u00ad"+
+    "O\u0000\u05b1\u05b2\u0001\u0000\u0000\u0000\u05b2\u05b3\u0006\u00c2\u000b"+
+    "\u0000\u05b3\u05b4\u0006\u00c2\u0000\u0000\u05b4\u05b5\u0006\u00c2\u001e"+
+    "\u0000\u05b5\u0194\u0001\u0000\u0000\u0000\u05b6\u05b7\u0003\u00a9M\u0000"+
+    "\u05b7\u05b8\u0001\u0000\u0000\u0000\u05b8\u05b9\u0006\u00c3\u000b\u0000"+
+    "\u05b9\u05ba\u0006\u00c3\u0000\u0000\u05ba\u05bb\u0006\u00c3\u001f\u0000"+
+    "\u05bb\u0196\u0001\u0000\u0000\u0000\u05bc\u05bd\u0003Y%\u0000\u05bd\u05be"+
+    "\u0001\u0000\u0000\u0000\u05be\u05bf\u0006\u00c4\u000b\u0000\u05bf\u05c0"+
+    "\u0006\u00c4\u0000\u0000\u05c0\u05c1\u0006\u00c4#\u0000\u05c1\u0198\u0001"+
+    "\u0000\u0000\u0000\u05c2\u05c3\u0003=\u0017\u0000\u05c3\u05c4\u0001\u0000"+
+    "\u0000\u0000\u05c4\u05c5\u0006\u00c5\u000f\u0000\u05c5\u05c6\u0006\u00c5"+
+    "\u000b\u0000\u05c6\u019a\u0001\u0000\u0000\u0000A\u0000\u0001\u0002\u0003"+
+    "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u0243\u024d\u0251\u0254"+
+    "\u025d\u025f\u026a\u027d\u0282\u028b\u0292\u0297\u0299\u02a4\u02ac\u02af"+
+    "\u02b1\u02b6\u02bb\u02c1\u02c8\u02cd\u02d3\u02d6\u02de\u02e2\u0366\u036b"+
+    "\u0372\u0374\u0384\u0389\u038e\u0390\u0396\u03e3\u03e8\u0419\u041d\u0422"+
+    "\u0427\u042c\u042e\u0432\u0434\u048b\u048f\u0494\u0523\u0525$\u0005\u0001"+
+    "\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005\u0003"+
+    "\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b\u0000"+
+    "\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0010\u0000\u0007"+
+    "A\u0000\u0005\u0000\u0000\u0007\u0018\u0000\u0007B\u0000\u0007h\u0000"+
+    "\u0007!\u0000\u0007\u001f\u0000\u0007L\u0000\u0007\u0019\u0000\u0007#"+
+    "\u0000\u0007/\u0000\u0007@\u0000\u0007P\u0000\u0005\n\u0000\u0005\u0007"+
     "\u0000\u0007Z\u0000\u0007Y\u0000\u0007D\u0000\u0007C\u0000\u0007X\u0000"+
-    "\u0005\f\u0000\u0005\u000e\u0000\u0007\u001d\u0000";
+    "\u0005\f\u0000\u0005\u000e\u0000\u0007\u001c\u0000";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
index eb3c70385d628..b52d842e79fb2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
@@ -23,7 +23,6 @@ null
 null
 null
 null
-null
 '|'
 null
 null
@@ -63,6 +62,7 @@ null
 '*'
 '/'
 '%'
+'match'
 null
 null
 ']'
@@ -141,7 +141,6 @@ STATS
 WHERE
 DEV_INLINESTATS
 DEV_LOOKUP
-DEV_MATCH
 DEV_METRICS
 UNKNOWN_CMD
 LINE_COMMENT
@@ -186,6 +185,7 @@ MINUS
 ASTERISK
 SLASH
 PERCENT
+MATCH
 NAMED_OR_POSITIONAL_PARAM
 OPENING_BRACKET
 CLOSING_BRACKET
@@ -257,6 +257,7 @@ valueExpression
 operatorExpression
 primaryExpression
 functionExpression
+functionName
 dataType
 rowCommand
 fields
@@ -271,6 +272,8 @@ deprecated_metadata
 metricsCommand
 evalCommand
 statsCommand
+aggFields
+aggField
 qualifiedName
 qualifiedNamePattern
 qualifiedNamePatterns
@@ -307,4 +310,4 @@ inlinestatsCommand
 
 
 atn:
-[4, 1, 120, 580, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 128, 8, 1, 10, 1, 12, 1, 131, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 139, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 157, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 169, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 176, 8, 5, 10, 5, 12, 5, 179, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 186, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 200, 8, 5, 10, 5, 12, 5, 203, 9, 5, 1, 6, 1, 6, 3, 6, 207, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 214, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 219, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 230, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 236, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 244, 8, 9, 10, 9, 12, 9, 247, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 257, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 262, 8, 10, 10, 10, 12, 10, 265, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 273, 8, 11, 10, 11, 12, 11, 276, 9, 11, 3, 11, 278, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 5, 14, 290, 8, 14, 10, 14, 12, 14, 293, 9, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 300, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 306, 8, 16, 10, 16, 12, 16, 309, 9, 16, 1, 16, 3, 16, 312, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 319, 8, 17, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 3, 20, 327, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 333, 8, 21, 10, 21, 12, 21, 336, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 346, 8, 23, 10, 23, 12, 23, 349, 9, 23, 1, 23, 3, 23, 352, 8, 23, 1, 23, 1, 23, 3, 23, 356, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 3, 25, 363, 8, 25, 1, 25, 1, 25, 3, 25, 367, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 372, 8, 26, 10, 26, 12, 26, 375, 9, 26, 1, 27, 1, 27, 1, 27, 5, 27, 380, 8, 27, 10, 27, 12, 27, 383, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 388, 8, 28, 10, 28, 12, 28, 391, 9, 28, 1, 29, 1, 29, 1, 30, 1, 30, 3, 30, 397, 8, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 412, 8, 31, 10, 31, 12, 31, 415, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 423, 8, 31, 10, 31, 12, 31, 426, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 434, 8, 31, 10, 31, 12, 31, 437, 9, 31, 1, 31, 1, 31, 3, 31, 441, 8, 31, 1, 32, 1, 32, 3, 32, 445, 8, 32, 1, 33, 1, 33, 3, 33, 449, 8, 33, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 458, 8, 35, 10, 35, 12, 35, 461, 9, 35, 1, 36, 1, 36, 3, 36, 465, 8, 36, 1, 36, 1, 36, 3, 36, 469, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 5, 39, 481, 8, 39, 10, 39, 12, 39, 484, 9, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 494, 8, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 5, 44, 506, 8, 44, 10, 44, 12, 44, 509, 9, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 3, 47, 519, 8, 47, 1, 48, 3, 48, 522, 8, 48, 1, 48, 1, 48, 1, 49, 3, 49, 527, 8, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 3, 55, 549, 8, 55, 1, 55, 1, 55, 1, 55, 1, 55, 5, 55, 555, 8, 55, 10, 55, 12, 55, 558, 9, 55, 3, 55, 560, 8, 55, 1, 56, 1, 56, 1, 56, 3, 56, 565, 8, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 578, 8, 58, 1, 58, 0, 4, 2, 10, 18, 20, 59, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 0, 8, 1, 0, 59, 60, 1, 0, 61, 63, 2, 0, 26, 26, 76, 76, 1, 0, 67, 68, 2, 0, 31, 31, 35, 35, 2, 0, 38, 38, 41, 41, 2, 0, 37, 37, 51, 51, 2, 0, 52, 52, 54, 58, 606, 0, 118, 1, 0, 0, 0, 2, 121, 1, 0, 0, 0, 4, 138, 1, 0, 0, 0, 6, 156, 1, 0, 0, 0, 8, 158, 1, 0, 0, 0, 10, 191, 1, 0, 0, 0, 12, 218, 1, 0, 0, 0, 14, 220, 1, 0, 0, 0, 16, 229, 1, 0, 0, 0, 18, 235, 1, 0, 0, 0, 20, 256, 1, 0, 0, 0, 22, 266, 1, 0, 0, 0, 24, 281, 1, 0, 0, 0, 26, 283, 1, 0, 0, 0, 28, 286, 1, 0, 0, 0, 30, 299, 1, 0, 0, 0, 32, 301, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 320, 1, 0, 0, 0, 38, 322, 1, 0, 0, 0, 40, 326, 1, 0, 0, 0, 42, 328, 1, 0, 0, 0, 44, 337, 1, 0, 0, 0, 46, 341, 1, 0, 0, 0, 48, 357, 1, 0, 0, 0, 50, 360, 1, 0, 0, 0, 52, 368, 1, 0, 0, 0, 54, 376, 1, 0, 0, 0, 56, 384, 1, 0, 0, 0, 58, 392, 1, 0, 0, 0, 60, 396, 1, 0, 0, 0, 62, 440, 1, 0, 0, 0, 64, 444, 1, 0, 0, 0, 66, 448, 1, 0, 0, 0, 68, 450, 1, 0, 0, 0, 70, 453, 1, 0, 0, 0, 72, 462, 1, 0, 0, 0, 74, 470, 1, 0, 0, 0, 76, 473, 1, 0, 0, 0, 78, 476, 1, 0, 0, 0, 80, 485, 1, 0, 0, 0, 82, 489, 1, 0, 0, 0, 84, 495, 1, 0, 0, 0, 86, 499, 1, 0, 0, 0, 88, 502, 1, 0, 0, 0, 90, 510, 1, 0, 0, 0, 92, 514, 1, 0, 0, 0, 94, 518, 1, 0, 0, 0, 96, 521, 1, 0, 0, 0, 98, 526, 1, 0, 0, 0, 100, 530, 1, 0, 0, 0, 102, 532, 1, 0, 0, 0, 104, 534, 1, 0, 0, 0, 106, 537, 1, 0, 0, 0, 108, 541, 1, 0, 0, 0, 110, 544, 1, 0, 0, 0, 112, 564, 1, 0, 0, 0, 114, 568, 1, 0, 0, 0, 116, 573, 1, 0, 0, 0, 118, 119, 3, 2, 1, 0, 119, 120, 5, 0, 0, 1, 120, 1, 1, 0, 0, 0, 121, 122, 6, 1, -1, 0, 122, 123, 3, 4, 2, 0, 123, 129, 1, 0, 0, 0, 124, 125, 10, 1, 0, 0, 125, 126, 5, 25, 0, 0, 126, 128, 3, 6, 3, 0, 127, 124, 1, 0, 0, 0, 128, 131, 1, 0, 0, 0, 129, 127, 1, 0, 0, 0, 129, 130, 1, 0, 0, 0, 130, 3, 1, 0, 0, 0, 131, 129, 1, 0, 0, 0, 132, 139, 3, 104, 52, 0, 133, 139, 3, 32, 16, 0, 134, 139, 3, 26, 13, 0, 135, 139, 3, 108, 54, 0, 136, 137, 4, 2, 1, 0, 137, 139, 3, 46, 23, 0, 138, 132, 1, 0, 0, 0, 138, 133, 1, 0, 0, 0, 138, 134, 1, 0, 0, 0, 138, 135, 1, 0, 0, 0, 138, 136, 1, 0, 0, 0, 139, 5, 1, 0, 0, 0, 140, 157, 3, 48, 24, 0, 141, 157, 3, 8, 4, 0, 142, 157, 3, 74, 37, 0, 143, 157, 3, 68, 34, 0, 144, 157, 3, 50, 25, 0, 145, 157, 3, 70, 35, 0, 146, 157, 3, 76, 38, 0, 147, 157, 3, 78, 39, 0, 148, 157, 3, 82, 41, 0, 149, 157, 3, 84, 42, 0, 150, 157, 3, 110, 55, 0, 151, 157, 3, 86, 43, 0, 152, 153, 4, 3, 2, 0, 153, 157, 3, 116, 58, 0, 154, 155, 4, 3, 3, 0, 155, 157, 3, 114, 57, 0, 156, 140, 1, 0, 0, 0, 156, 141, 1, 0, 0, 0, 156, 142, 1, 0, 0, 0, 156, 143, 1, 0, 0, 0, 156, 144, 1, 0, 0, 0, 156, 145, 1, 0, 0, 0, 156, 146, 1, 0, 0, 0, 156, 147, 1, 0, 0, 0, 156, 148, 1, 0, 0, 0, 156, 149, 1, 0, 0, 0, 156, 150, 1, 0, 0, 0, 156, 151, 1, 0, 0, 0, 156, 152, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 157, 7, 1, 0, 0, 0, 158, 159, 5, 16, 0, 0, 159, 160, 3, 10, 5, 0, 160, 9, 1, 0, 0, 0, 161, 162, 6, 5, -1, 0, 162, 163, 5, 44, 0, 0, 163, 192, 3, 10, 5, 8, 164, 192, 3, 16, 8, 0, 165, 192, 3, 12, 6, 0, 166, 168, 3, 16, 8, 0, 167, 169, 5, 44, 0, 0, 168, 167, 1, 0, 0, 0, 168, 169, 1, 0, 0, 0, 169, 170, 1, 0, 0, 0, 170, 171, 5, 39, 0, 0, 171, 172, 5, 43, 0, 0, 172, 177, 3, 16, 8, 0, 173, 174, 5, 34, 0, 0, 174, 176, 3, 16, 8, 0, 175, 173, 1, 0, 0, 0, 176, 179, 1, 0, 0, 0, 177, 175, 1, 0, 0, 0, 177, 178, 1, 0, 0, 0, 178, 180, 1, 0, 0, 0, 179, 177, 1, 0, 0, 0, 180, 181, 5, 50, 0, 0, 181, 192, 1, 0, 0, 0, 182, 183, 3, 16, 8, 0, 183, 185, 5, 40, 0, 0, 184, 186, 5, 44, 0, 0, 185, 184, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 1, 0, 0, 0, 187, 188, 5, 45, 0, 0, 188, 192, 1, 0, 0, 0, 189, 190, 4, 5, 4, 0, 190, 192, 3, 14, 7, 0, 191, 161, 1, 0, 0, 0, 191, 164, 1, 0, 0, 0, 191, 165, 1, 0, 0, 0, 191, 166, 1, 0, 0, 0, 191, 182, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 201, 1, 0, 0, 0, 193, 194, 10, 5, 0, 0, 194, 195, 5, 30, 0, 0, 195, 200, 3, 10, 5, 6, 196, 197, 10, 4, 0, 0, 197, 198, 5, 47, 0, 0, 198, 200, 3, 10, 5, 5, 199, 193, 1, 0, 0, 0, 199, 196, 1, 0, 0, 0, 200, 203, 1, 0, 0, 0, 201, 199, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 11, 1, 0, 0, 0, 203, 201, 1, 0, 0, 0, 204, 206, 3, 16, 8, 0, 205, 207, 5, 44, 0, 0, 206, 205, 1, 0, 0, 0, 206, 207, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 209, 5, 42, 0, 0, 209, 210, 3, 100, 50, 0, 210, 219, 1, 0, 0, 0, 211, 213, 3, 16, 8, 0, 212, 214, 5, 44, 0, 0, 213, 212, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 216, 5, 49, 0, 0, 216, 217, 3, 100, 50, 0, 217, 219, 1, 0, 0, 0, 218, 204, 1, 0, 0, 0, 218, 211, 1, 0, 0, 0, 219, 13, 1, 0, 0, 0, 220, 221, 3, 16, 8, 0, 221, 222, 5, 19, 0, 0, 222, 223, 3, 100, 50, 0, 223, 15, 1, 0, 0, 0, 224, 230, 3, 18, 9, 0, 225, 226, 3, 18, 9, 0, 226, 227, 3, 102, 51, 0, 227, 228, 3, 18, 9, 0, 228, 230, 1, 0, 0, 0, 229, 224, 1, 0, 0, 0, 229, 225, 1, 0, 0, 0, 230, 17, 1, 0, 0, 0, 231, 232, 6, 9, -1, 0, 232, 236, 3, 20, 10, 0, 233, 234, 7, 0, 0, 0, 234, 236, 3, 18, 9, 3, 235, 231, 1, 0, 0, 0, 235, 233, 1, 0, 0, 0, 236, 245, 1, 0, 0, 0, 237, 238, 10, 2, 0, 0, 238, 239, 7, 1, 0, 0, 239, 244, 3, 18, 9, 3, 240, 241, 10, 1, 0, 0, 241, 242, 7, 0, 0, 0, 242, 244, 3, 18, 9, 2, 243, 237, 1, 0, 0, 0, 243, 240, 1, 0, 0, 0, 244, 247, 1, 0, 0, 0, 245, 243, 1, 0, 0, 0, 245, 246, 1, 0, 0, 0, 246, 19, 1, 0, 0, 0, 247, 245, 1, 0, 0, 0, 248, 249, 6, 10, -1, 0, 249, 257, 3, 62, 31, 0, 250, 257, 3, 52, 26, 0, 251, 257, 3, 22, 11, 0, 252, 253, 5, 43, 0, 0, 253, 254, 3, 10, 5, 0, 254, 255, 5, 50, 0, 0, 255, 257, 1, 0, 0, 0, 256, 248, 1, 0, 0, 0, 256, 250, 1, 0, 0, 0, 256, 251, 1, 0, 0, 0, 256, 252, 1, 0, 0, 0, 257, 263, 1, 0, 0, 0, 258, 259, 10, 1, 0, 0, 259, 260, 5, 33, 0, 0, 260, 262, 3, 24, 12, 0, 261, 258, 1, 0, 0, 0, 262, 265, 1, 0, 0, 0, 263, 261, 1, 0, 0, 0, 263, 264, 1, 0, 0, 0, 264, 21, 1, 0, 0, 0, 265, 263, 1, 0, 0, 0, 266, 267, 3, 66, 33, 0, 267, 277, 5, 43, 0, 0, 268, 278, 5, 61, 0, 0, 269, 274, 3, 10, 5, 0, 270, 271, 5, 34, 0, 0, 271, 273, 3, 10, 5, 0, 272, 270, 1, 0, 0, 0, 273, 276, 1, 0, 0, 0, 274, 272, 1, 0, 0, 0, 274, 275, 1, 0, 0, 0, 275, 278, 1, 0, 0, 0, 276, 274, 1, 0, 0, 0, 277, 268, 1, 0, 0, 0, 277, 269, 1, 0, 0, 0, 277, 278, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 280, 5, 50, 0, 0, 280, 23, 1, 0, 0, 0, 281, 282, 3, 58, 29, 0, 282, 25, 1, 0, 0, 0, 283, 284, 5, 12, 0, 0, 284, 285, 3, 28, 14, 0, 285, 27, 1, 0, 0, 0, 286, 291, 3, 30, 15, 0, 287, 288, 5, 34, 0, 0, 288, 290, 3, 30, 15, 0, 289, 287, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 291, 292, 1, 0, 0, 0, 292, 29, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 294, 300, 3, 10, 5, 0, 295, 296, 3, 52, 26, 0, 296, 297, 5, 32, 0, 0, 297, 298, 3, 10, 5, 0, 298, 300, 1, 0, 0, 0, 299, 294, 1, 0, 0, 0, 299, 295, 1, 0, 0, 0, 300, 31, 1, 0, 0, 0, 301, 302, 5, 6, 0, 0, 302, 307, 3, 34, 17, 0, 303, 304, 5, 34, 0, 0, 304, 306, 3, 34, 17, 0, 305, 303, 1, 0, 0, 0, 306, 309, 1, 0, 0, 0, 307, 305, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 311, 1, 0, 0, 0, 309, 307, 1, 0, 0, 0, 310, 312, 3, 40, 20, 0, 311, 310, 1, 0, 0, 0, 311, 312, 1, 0, 0, 0, 312, 33, 1, 0, 0, 0, 313, 314, 3, 36, 18, 0, 314, 315, 5, 104, 0, 0, 315, 316, 3, 38, 19, 0, 316, 319, 1, 0, 0, 0, 317, 319, 3, 38, 19, 0, 318, 313, 1, 0, 0, 0, 318, 317, 1, 0, 0, 0, 319, 35, 1, 0, 0, 0, 320, 321, 5, 76, 0, 0, 321, 37, 1, 0, 0, 0, 322, 323, 7, 2, 0, 0, 323, 39, 1, 0, 0, 0, 324, 327, 3, 42, 21, 0, 325, 327, 3, 44, 22, 0, 326, 324, 1, 0, 0, 0, 326, 325, 1, 0, 0, 0, 327, 41, 1, 0, 0, 0, 328, 329, 5, 75, 0, 0, 329, 334, 5, 76, 0, 0, 330, 331, 5, 34, 0, 0, 331, 333, 5, 76, 0, 0, 332, 330, 1, 0, 0, 0, 333, 336, 1, 0, 0, 0, 334, 332, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 43, 1, 0, 0, 0, 336, 334, 1, 0, 0, 0, 337, 338, 5, 65, 0, 0, 338, 339, 3, 42, 21, 0, 339, 340, 5, 66, 0, 0, 340, 45, 1, 0, 0, 0, 341, 342, 5, 20, 0, 0, 342, 347, 3, 34, 17, 0, 343, 344, 5, 34, 0, 0, 344, 346, 3, 34, 17, 0, 345, 343, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 351, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 350, 352, 3, 28, 14, 0, 351, 350, 1, 0, 0, 0, 351, 352, 1, 0, 0, 0, 352, 355, 1, 0, 0, 0, 353, 354, 5, 29, 0, 0, 354, 356, 3, 28, 14, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 47, 1, 0, 0, 0, 357, 358, 5, 4, 0, 0, 358, 359, 3, 28, 14, 0, 359, 49, 1, 0, 0, 0, 360, 362, 5, 15, 0, 0, 361, 363, 3, 28, 14, 0, 362, 361, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 366, 1, 0, 0, 0, 364, 365, 5, 29, 0, 0, 365, 367, 3, 28, 14, 0, 366, 364, 1, 0, 0, 0, 366, 367, 1, 0, 0, 0, 367, 51, 1, 0, 0, 0, 368, 373, 3, 66, 33, 0, 369, 370, 5, 36, 0, 0, 370, 372, 3, 66, 33, 0, 371, 369, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 371, 1, 0, 0, 0, 373, 374, 1, 0, 0, 0, 374, 53, 1, 0, 0, 0, 375, 373, 1, 0, 0, 0, 376, 381, 3, 60, 30, 0, 377, 378, 5, 36, 0, 0, 378, 380, 3, 60, 30, 0, 379, 377, 1, 0, 0, 0, 380, 383, 1, 0, 0, 0, 381, 379, 1, 0, 0, 0, 381, 382, 1, 0, 0, 0, 382, 55, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 384, 389, 3, 54, 27, 0, 385, 386, 5, 34, 0, 0, 386, 388, 3, 54, 27, 0, 387, 385, 1, 0, 0, 0, 388, 391, 1, 0, 0, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 57, 1, 0, 0, 0, 391, 389, 1, 0, 0, 0, 392, 393, 7, 3, 0, 0, 393, 59, 1, 0, 0, 0, 394, 397, 5, 80, 0, 0, 395, 397, 3, 64, 32, 0, 396, 394, 1, 0, 0, 0, 396, 395, 1, 0, 0, 0, 397, 61, 1, 0, 0, 0, 398, 441, 5, 45, 0, 0, 399, 400, 3, 98, 49, 0, 400, 401, 5, 67, 0, 0, 401, 441, 1, 0, 0, 0, 402, 441, 3, 96, 48, 0, 403, 441, 3, 98, 49, 0, 404, 441, 3, 92, 46, 0, 405, 441, 3, 64, 32, 0, 406, 441, 3, 100, 50, 0, 407, 408, 5, 65, 0, 0, 408, 413, 3, 94, 47, 0, 409, 410, 5, 34, 0, 0, 410, 412, 3, 94, 47, 0, 411, 409, 1, 0, 0, 0, 412, 415, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 413, 414, 1, 0, 0, 0, 414, 416, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 416, 417, 5, 66, 0, 0, 417, 441, 1, 0, 0, 0, 418, 419, 5, 65, 0, 0, 419, 424, 3, 92, 46, 0, 420, 421, 5, 34, 0, 0, 421, 423, 3, 92, 46, 0, 422, 420, 1, 0, 0, 0, 423, 426, 1, 0, 0, 0, 424, 422, 1, 0, 0, 0, 424, 425, 1, 0, 0, 0, 425, 427, 1, 0, 0, 0, 426, 424, 1, 0, 0, 0, 427, 428, 5, 66, 0, 0, 428, 441, 1, 0, 0, 0, 429, 430, 5, 65, 0, 0, 430, 435, 3, 100, 50, 0, 431, 432, 5, 34, 0, 0, 432, 434, 3, 100, 50, 0, 433, 431, 1, 0, 0, 0, 434, 437, 1, 0, 0, 0, 435, 433, 1, 0, 0, 0, 435, 436, 1, 0, 0, 0, 436, 438, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 438, 439, 5, 66, 0, 0, 439, 441, 1, 0, 0, 0, 440, 398, 1, 0, 0, 0, 440, 399, 1, 0, 0, 0, 440, 402, 1, 0, 0, 0, 440, 403, 1, 0, 0, 0, 440, 404, 1, 0, 0, 0, 440, 405, 1, 0, 0, 0, 440, 406, 1, 0, 0, 0, 440, 407, 1, 0, 0, 0, 440, 418, 1, 0, 0, 0, 440, 429, 1, 0, 0, 0, 441, 63, 1, 0, 0, 0, 442, 445, 5, 48, 0, 0, 443, 445, 5, 64, 0, 0, 444, 442, 1, 0, 0, 0, 444, 443, 1, 0, 0, 0, 445, 65, 1, 0, 0, 0, 446, 449, 3, 58, 29, 0, 447, 449, 3, 64, 32, 0, 448, 446, 1, 0, 0, 0, 448, 447, 1, 0, 0, 0, 449, 67, 1, 0, 0, 0, 450, 451, 5, 9, 0, 0, 451, 452, 5, 27, 0, 0, 452, 69, 1, 0, 0, 0, 453, 454, 5, 14, 0, 0, 454, 459, 3, 72, 36, 0, 455, 456, 5, 34, 0, 0, 456, 458, 3, 72, 36, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 71, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 464, 3, 10, 5, 0, 463, 465, 7, 4, 0, 0, 464, 463, 1, 0, 0, 0, 464, 465, 1, 0, 0, 0, 465, 468, 1, 0, 0, 0, 466, 467, 5, 46, 0, 0, 467, 469, 7, 5, 0, 0, 468, 466, 1, 0, 0, 0, 468, 469, 1, 0, 0, 0, 469, 73, 1, 0, 0, 0, 470, 471, 5, 8, 0, 0, 471, 472, 3, 56, 28, 0, 472, 75, 1, 0, 0, 0, 473, 474, 5, 2, 0, 0, 474, 475, 3, 56, 28, 0, 475, 77, 1, 0, 0, 0, 476, 477, 5, 11, 0, 0, 477, 482, 3, 80, 40, 0, 478, 479, 5, 34, 0, 0, 479, 481, 3, 80, 40, 0, 480, 478, 1, 0, 0, 0, 481, 484, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 482, 483, 1, 0, 0, 0, 483, 79, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 486, 3, 54, 27, 0, 486, 487, 5, 84, 0, 0, 487, 488, 3, 54, 27, 0, 488, 81, 1, 0, 0, 0, 489, 490, 5, 1, 0, 0, 490, 491, 3, 20, 10, 0, 491, 493, 3, 100, 50, 0, 492, 494, 3, 88, 44, 0, 493, 492, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 83, 1, 0, 0, 0, 495, 496, 5, 7, 0, 0, 496, 497, 3, 20, 10, 0, 497, 498, 3, 100, 50, 0, 498, 85, 1, 0, 0, 0, 499, 500, 5, 10, 0, 0, 500, 501, 3, 52, 26, 0, 501, 87, 1, 0, 0, 0, 502, 507, 3, 90, 45, 0, 503, 504, 5, 34, 0, 0, 504, 506, 3, 90, 45, 0, 505, 503, 1, 0, 0, 0, 506, 509, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 89, 1, 0, 0, 0, 509, 507, 1, 0, 0, 0, 510, 511, 3, 58, 29, 0, 511, 512, 5, 32, 0, 0, 512, 513, 3, 62, 31, 0, 513, 91, 1, 0, 0, 0, 514, 515, 7, 6, 0, 0, 515, 93, 1, 0, 0, 0, 516, 519, 3, 96, 48, 0, 517, 519, 3, 98, 49, 0, 518, 516, 1, 0, 0, 0, 518, 517, 1, 0, 0, 0, 519, 95, 1, 0, 0, 0, 520, 522, 7, 0, 0, 0, 521, 520, 1, 0, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 1, 0, 0, 0, 523, 524, 5, 28, 0, 0, 524, 97, 1, 0, 0, 0, 525, 527, 7, 0, 0, 0, 526, 525, 1, 0, 0, 0, 526, 527, 1, 0, 0, 0, 527, 528, 1, 0, 0, 0, 528, 529, 5, 27, 0, 0, 529, 99, 1, 0, 0, 0, 530, 531, 5, 26, 0, 0, 531, 101, 1, 0, 0, 0, 532, 533, 7, 7, 0, 0, 533, 103, 1, 0, 0, 0, 534, 535, 5, 5, 0, 0, 535, 536, 3, 106, 53, 0, 536, 105, 1, 0, 0, 0, 537, 538, 5, 65, 0, 0, 538, 539, 3, 2, 1, 0, 539, 540, 5, 66, 0, 0, 540, 107, 1, 0, 0, 0, 541, 542, 5, 13, 0, 0, 542, 543, 5, 100, 0, 0, 543, 109, 1, 0, 0, 0, 544, 545, 5, 3, 0, 0, 545, 548, 5, 90, 0, 0, 546, 547, 5, 88, 0, 0, 547, 549, 3, 54, 27, 0, 548, 546, 1, 0, 0, 0, 548, 549, 1, 0, 0, 0, 549, 559, 1, 0, 0, 0, 550, 551, 5, 89, 0, 0, 551, 556, 3, 112, 56, 0, 552, 553, 5, 34, 0, 0, 553, 555, 3, 112, 56, 0, 554, 552, 1, 0, 0, 0, 555, 558, 1, 0, 0, 0, 556, 554, 1, 0, 0, 0, 556, 557, 1, 0, 0, 0, 557, 560, 1, 0, 0, 0, 558, 556, 1, 0, 0, 0, 559, 550, 1, 0, 0, 0, 559, 560, 1, 0, 0, 0, 560, 111, 1, 0, 0, 0, 561, 562, 3, 54, 27, 0, 562, 563, 5, 32, 0, 0, 563, 565, 1, 0, 0, 0, 564, 561, 1, 0, 0, 0, 564, 565, 1, 0, 0, 0, 565, 566, 1, 0, 0, 0, 566, 567, 3, 54, 27, 0, 567, 113, 1, 0, 0, 0, 568, 569, 5, 18, 0, 0, 569, 570, 3, 34, 17, 0, 570, 571, 5, 88, 0, 0, 571, 572, 3, 56, 28, 0, 572, 115, 1, 0, 0, 0, 573, 574, 5, 17, 0, 0, 574, 577, 3, 28, 14, 0, 575, 576, 5, 29, 0, 0, 576, 578, 3, 28, 14, 0, 577, 575, 1, 0, 0, 0, 577, 578, 1, 0, 0, 0, 578, 117, 1, 0, 0, 0, 56, 129, 138, 156, 168, 177, 185, 191, 199, 201, 206, 213, 218, 229, 235, 243, 245, 256, 263, 274, 277, 291, 299, 307, 311, 318, 326, 334, 347, 351, 355, 362, 366, 373, 381, 389, 396, 413, 424, 435, 440, 444, 448, 459, 464, 468, 482, 493, 507, 518, 521, 526, 548, 556, 559, 564, 577]
\ No newline at end of file
+[4, 1, 120, 605, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 134, 8, 1, 10, 1, 12, 1, 137, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 145, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 163, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 175, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 182, 8, 5, 10, 5, 12, 5, 185, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 206, 8, 5, 10, 5, 12, 5, 209, 9, 5, 1, 6, 1, 6, 3, 6, 213, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 220, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 236, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 242, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 250, 8, 9, 10, 9, 12, 9, 253, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 263, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 268, 8, 10, 10, 10, 12, 10, 271, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 279, 8, 11, 10, 11, 12, 11, 282, 9, 11, 3, 11, 284, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 3, 12, 290, 8, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 300, 8, 15, 10, 15, 12, 15, 303, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 308, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 316, 8, 17, 10, 17, 12, 17, 319, 9, 17, 1, 17, 3, 17, 322, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 327, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 337, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 343, 8, 22, 10, 22, 12, 22, 346, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 356, 8, 24, 10, 24, 12, 24, 359, 9, 24, 1, 24, 3, 24, 362, 8, 24, 1, 24, 1, 24, 3, 24, 366, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 373, 8, 26, 1, 26, 1, 26, 3, 26, 377, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 382, 8, 27, 10, 27, 12, 27, 385, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 390, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 395, 8, 29, 10, 29, 12, 29, 398, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 403, 8, 30, 10, 30, 12, 30, 406, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 411, 8, 31, 10, 31, 12, 31, 414, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 421, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 436, 8, 34, 10, 34, 12, 34, 439, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 3, 34, 465, 8, 34, 1, 35, 1, 35, 3, 35, 469, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 474, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 483, 8, 38, 10, 38, 12, 38, 486, 9, 38, 1, 39, 1, 39, 3, 39, 490, 8, 39, 1, 39, 1, 39, 3, 39, 494, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 506, 8, 42, 10, 42, 12, 42, 509, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 519, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 531, 8, 47, 10, 47, 12, 47, 534, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 544, 8, 50, 1, 51, 3, 51, 547, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 552, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 574, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 580, 8, 58, 10, 58, 12, 58, 583, 9, 58, 3, 58, 585, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 590, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 603, 8, 61, 1, 61, 0, 4, 2, 10, 18, 20, 62, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 0, 8, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 25, 25, 76, 76, 1, 0, 67, 68, 2, 0, 30, 30, 34, 34, 2, 0, 37, 37, 40, 40, 2, 0, 36, 36, 50, 50, 2, 0, 51, 51, 53, 57, 631, 0, 124, 1, 0, 0, 0, 2, 127, 1, 0, 0, 0, 4, 144, 1, 0, 0, 0, 6, 162, 1, 0, 0, 0, 8, 164, 1, 0, 0, 0, 10, 197, 1, 0, 0, 0, 12, 224, 1, 0, 0, 0, 14, 226, 1, 0, 0, 0, 16, 235, 1, 0, 0, 0, 18, 241, 1, 0, 0, 0, 20, 262, 1, 0, 0, 0, 22, 272, 1, 0, 0, 0, 24, 289, 1, 0, 0, 0, 26, 291, 1, 0, 0, 0, 28, 293, 1, 0, 0, 0, 30, 296, 1, 0, 0, 0, 32, 307, 1, 0, 0, 0, 34, 311, 1, 0, 0, 0, 36, 326, 1, 0, 0, 0, 38, 330, 1, 0, 0, 0, 40, 332, 1, 0, 0, 0, 42, 336, 1, 0, 0, 0, 44, 338, 1, 0, 0, 0, 46, 347, 1, 0, 0, 0, 48, 351, 1, 0, 0, 0, 50, 367, 1, 0, 0, 0, 52, 370, 1, 0, 0, 0, 54, 378, 1, 0, 0, 0, 56, 386, 1, 0, 0, 0, 58, 391, 1, 0, 0, 0, 60, 399, 1, 0, 0, 0, 62, 407, 1, 0, 0, 0, 64, 415, 1, 0, 0, 0, 66, 420, 1, 0, 0, 0, 68, 464, 1, 0, 0, 0, 70, 468, 1, 0, 0, 0, 72, 473, 1, 0, 0, 0, 74, 475, 1, 0, 0, 0, 76, 478, 1, 0, 0, 0, 78, 487, 1, 0, 0, 0, 80, 495, 1, 0, 0, 0, 82, 498, 1, 0, 0, 0, 84, 501, 1, 0, 0, 0, 86, 510, 1, 0, 0, 0, 88, 514, 1, 0, 0, 0, 90, 520, 1, 0, 0, 0, 92, 524, 1, 0, 0, 0, 94, 527, 1, 0, 0, 0, 96, 535, 1, 0, 0, 0, 98, 539, 1, 0, 0, 0, 100, 543, 1, 0, 0, 0, 102, 546, 1, 0, 0, 0, 104, 551, 1, 0, 0, 0, 106, 555, 1, 0, 0, 0, 108, 557, 1, 0, 0, 0, 110, 559, 1, 0, 0, 0, 112, 562, 1, 0, 0, 0, 114, 566, 1, 0, 0, 0, 116, 569, 1, 0, 0, 0, 118, 589, 1, 0, 0, 0, 120, 593, 1, 0, 0, 0, 122, 598, 1, 0, 0, 0, 124, 125, 3, 2, 1, 0, 125, 126, 5, 0, 0, 1, 126, 1, 1, 0, 0, 0, 127, 128, 6, 1, -1, 0, 128, 129, 3, 4, 2, 0, 129, 135, 1, 0, 0, 0, 130, 131, 10, 1, 0, 0, 131, 132, 5, 24, 0, 0, 132, 134, 3, 6, 3, 0, 133, 130, 1, 0, 0, 0, 134, 137, 1, 0, 0, 0, 135, 133, 1, 0, 0, 0, 135, 136, 1, 0, 0, 0, 136, 3, 1, 0, 0, 0, 137, 135, 1, 0, 0, 0, 138, 145, 3, 110, 55, 0, 139, 145, 3, 34, 17, 0, 140, 145, 3, 28, 14, 0, 141, 145, 3, 114, 57, 0, 142, 143, 4, 2, 1, 0, 143, 145, 3, 48, 24, 0, 144, 138, 1, 0, 0, 0, 144, 139, 1, 0, 0, 0, 144, 140, 1, 0, 0, 0, 144, 141, 1, 0, 0, 0, 144, 142, 1, 0, 0, 0, 145, 5, 1, 0, 0, 0, 146, 163, 3, 50, 25, 0, 147, 163, 3, 8, 4, 0, 148, 163, 3, 80, 40, 0, 149, 163, 3, 74, 37, 0, 150, 163, 3, 52, 26, 0, 151, 163, 3, 76, 38, 0, 152, 163, 3, 82, 41, 0, 153, 163, 3, 84, 42, 0, 154, 163, 3, 88, 44, 0, 155, 163, 3, 90, 45, 0, 156, 163, 3, 116, 58, 0, 157, 163, 3, 92, 46, 0, 158, 159, 4, 3, 2, 0, 159, 163, 3, 122, 61, 0, 160, 161, 4, 3, 3, 0, 161, 163, 3, 120, 60, 0, 162, 146, 1, 0, 0, 0, 162, 147, 1, 0, 0, 0, 162, 148, 1, 0, 0, 0, 162, 149, 1, 0, 0, 0, 162, 150, 1, 0, 0, 0, 162, 151, 1, 0, 0, 0, 162, 152, 1, 0, 0, 0, 162, 153, 1, 0, 0, 0, 162, 154, 1, 0, 0, 0, 162, 155, 1, 0, 0, 0, 162, 156, 1, 0, 0, 0, 162, 157, 1, 0, 0, 0, 162, 158, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 163, 7, 1, 0, 0, 0, 164, 165, 5, 16, 0, 0, 165, 166, 3, 10, 5, 0, 166, 9, 1, 0, 0, 0, 167, 168, 6, 5, -1, 0, 168, 169, 5, 43, 0, 0, 169, 198, 3, 10, 5, 8, 170, 198, 3, 16, 8, 0, 171, 198, 3, 12, 6, 0, 172, 174, 3, 16, 8, 0, 173, 175, 5, 43, 0, 0, 174, 173, 1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 5, 38, 0, 0, 177, 178, 5, 42, 0, 0, 178, 183, 3, 16, 8, 0, 179, 180, 5, 33, 0, 0, 180, 182, 3, 16, 8, 0, 181, 179, 1, 0, 0, 0, 182, 185, 1, 0, 0, 0, 183, 181, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 183, 1, 0, 0, 0, 186, 187, 5, 49, 0, 0, 187, 198, 1, 0, 0, 0, 188, 189, 3, 16, 8, 0, 189, 191, 5, 39, 0, 0, 190, 192, 5, 43, 0, 0, 191, 190, 1, 0, 0, 0, 191, 192, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193, 194, 5, 44, 0, 0, 194, 198, 1, 0, 0, 0, 195, 196, 4, 5, 4, 0, 196, 198, 3, 14, 7, 0, 197, 167, 1, 0, 0, 0, 197, 170, 1, 0, 0, 0, 197, 171, 1, 0, 0, 0, 197, 172, 1, 0, 0, 0, 197, 188, 1, 0, 0, 0, 197, 195, 1, 0, 0, 0, 198, 207, 1, 0, 0, 0, 199, 200, 10, 5, 0, 0, 200, 201, 5, 29, 0, 0, 201, 206, 3, 10, 5, 6, 202, 203, 10, 4, 0, 0, 203, 204, 5, 46, 0, 0, 204, 206, 3, 10, 5, 5, 205, 199, 1, 0, 0, 0, 205, 202, 1, 0, 0, 0, 206, 209, 1, 0, 0, 0, 207, 205, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 11, 1, 0, 0, 0, 209, 207, 1, 0, 0, 0, 210, 212, 3, 16, 8, 0, 211, 213, 5, 43, 0, 0, 212, 211, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 5, 41, 0, 0, 215, 216, 3, 106, 53, 0, 216, 225, 1, 0, 0, 0, 217, 219, 3, 16, 8, 0, 218, 220, 5, 43, 0, 0, 219, 218, 1, 0, 0, 0, 219, 220, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 5, 48, 0, 0, 222, 223, 3, 106, 53, 0, 223, 225, 1, 0, 0, 0, 224, 210, 1, 0, 0, 0, 224, 217, 1, 0, 0, 0, 225, 13, 1, 0, 0, 0, 226, 227, 3, 16, 8, 0, 227, 228, 5, 63, 0, 0, 228, 229, 3, 106, 53, 0, 229, 15, 1, 0, 0, 0, 230, 236, 3, 18, 9, 0, 231, 232, 3, 18, 9, 0, 232, 233, 3, 108, 54, 0, 233, 234, 3, 18, 9, 0, 234, 236, 1, 0, 0, 0, 235, 230, 1, 0, 0, 0, 235, 231, 1, 0, 0, 0, 236, 17, 1, 0, 0, 0, 237, 238, 6, 9, -1, 0, 238, 242, 3, 20, 10, 0, 239, 240, 7, 0, 0, 0, 240, 242, 3, 18, 9, 3, 241, 237, 1, 0, 0, 0, 241, 239, 1, 0, 0, 0, 242, 251, 1, 0, 0, 0, 243, 244, 10, 2, 0, 0, 244, 245, 7, 1, 0, 0, 245, 250, 3, 18, 9, 3, 246, 247, 10, 1, 0, 0, 247, 248, 7, 0, 0, 0, 248, 250, 3, 18, 9, 2, 249, 243, 1, 0, 0, 0, 249, 246, 1, 0, 0, 0, 250, 253, 1, 0, 0, 0, 251, 249, 1, 0, 0, 0, 251, 252, 1, 0, 0, 0, 252, 19, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 254, 255, 6, 10, -1, 0, 255, 263, 3, 68, 34, 0, 256, 263, 3, 58, 29, 0, 257, 263, 3, 22, 11, 0, 258, 259, 5, 42, 0, 0, 259, 260, 3, 10, 5, 0, 260, 261, 5, 49, 0, 0, 261, 263, 1, 0, 0, 0, 262, 254, 1, 0, 0, 0, 262, 256, 1, 0, 0, 0, 262, 257, 1, 0, 0, 0, 262, 258, 1, 0, 0, 0, 263, 269, 1, 0, 0, 0, 264, 265, 10, 1, 0, 0, 265, 266, 5, 32, 0, 0, 266, 268, 3, 26, 13, 0, 267, 264, 1, 0, 0, 0, 268, 271, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 21, 1, 0, 0, 0, 271, 269, 1, 0, 0, 0, 272, 273, 3, 24, 12, 0, 273, 283, 5, 42, 0, 0, 274, 284, 5, 60, 0, 0, 275, 280, 3, 10, 5, 0, 276, 277, 5, 33, 0, 0, 277, 279, 3, 10, 5, 0, 278, 276, 1, 0, 0, 0, 279, 282, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 274, 1, 0, 0, 0, 283, 275, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 285, 1, 0, 0, 0, 285, 286, 5, 49, 0, 0, 286, 23, 1, 0, 0, 0, 287, 290, 5, 63, 0, 0, 288, 290, 3, 72, 36, 0, 289, 287, 1, 0, 0, 0, 289, 288, 1, 0, 0, 0, 290, 25, 1, 0, 0, 0, 291, 292, 3, 64, 32, 0, 292, 27, 1, 0, 0, 0, 293, 294, 5, 12, 0, 0, 294, 295, 3, 30, 15, 0, 295, 29, 1, 0, 0, 0, 296, 301, 3, 32, 16, 0, 297, 298, 5, 33, 0, 0, 298, 300, 3, 32, 16, 0, 299, 297, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 31, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 3, 58, 29, 0, 305, 306, 5, 31, 0, 0, 306, 308, 1, 0, 0, 0, 307, 304, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 310, 3, 10, 5, 0, 310, 33, 1, 0, 0, 0, 311, 312, 5, 6, 0, 0, 312, 317, 3, 36, 18, 0, 313, 314, 5, 33, 0, 0, 314, 316, 3, 36, 18, 0, 315, 313, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 321, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 3, 42, 21, 0, 321, 320, 1, 0, 0, 0, 321, 322, 1, 0, 0, 0, 322, 35, 1, 0, 0, 0, 323, 324, 3, 38, 19, 0, 324, 325, 5, 104, 0, 0, 325, 327, 1, 0, 0, 0, 326, 323, 1, 0, 0, 0, 326, 327, 1, 0, 0, 0, 327, 328, 1, 0, 0, 0, 328, 329, 3, 40, 20, 0, 329, 37, 1, 0, 0, 0, 330, 331, 5, 76, 0, 0, 331, 39, 1, 0, 0, 0, 332, 333, 7, 2, 0, 0, 333, 41, 1, 0, 0, 0, 334, 337, 3, 44, 22, 0, 335, 337, 3, 46, 23, 0, 336, 334, 1, 0, 0, 0, 336, 335, 1, 0, 0, 0, 337, 43, 1, 0, 0, 0, 338, 339, 5, 75, 0, 0, 339, 344, 5, 76, 0, 0, 340, 341, 5, 33, 0, 0, 341, 343, 5, 76, 0, 0, 342, 340, 1, 0, 0, 0, 343, 346, 1, 0, 0, 0, 344, 342, 1, 0, 0, 0, 344, 345, 1, 0, 0, 0, 345, 45, 1, 0, 0, 0, 346, 344, 1, 0, 0, 0, 347, 348, 5, 65, 0, 0, 348, 349, 3, 44, 22, 0, 349, 350, 5, 66, 0, 0, 350, 47, 1, 0, 0, 0, 351, 352, 5, 19, 0, 0, 352, 357, 3, 36, 18, 0, 353, 354, 5, 33, 0, 0, 354, 356, 3, 36, 18, 0, 355, 353, 1, 0, 0, 0, 356, 359, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 361, 1, 0, 0, 0, 359, 357, 1, 0, 0, 0, 360, 362, 3, 54, 27, 0, 361, 360, 1, 0, 0, 0, 361, 362, 1, 0, 0, 0, 362, 365, 1, 0, 0, 0, 363, 364, 5, 28, 0, 0, 364, 366, 3, 30, 15, 0, 365, 363, 1, 0, 0, 0, 365, 366, 1, 0, 0, 0, 366, 49, 1, 0, 0, 0, 367, 368, 5, 4, 0, 0, 368, 369, 3, 30, 15, 0, 369, 51, 1, 0, 0, 0, 370, 372, 5, 15, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 28, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 53, 1, 0, 0, 0, 378, 383, 3, 56, 28, 0, 379, 380, 5, 33, 0, 0, 380, 382, 3, 56, 28, 0, 381, 379, 1, 0, 0, 0, 382, 385, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 55, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 386, 389, 3, 32, 16, 0, 387, 388, 5, 16, 0, 0, 388, 390, 3, 10, 5, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 57, 1, 0, 0, 0, 391, 396, 3, 72, 36, 0, 392, 393, 5, 35, 0, 0, 393, 395, 3, 72, 36, 0, 394, 392, 1, 0, 0, 0, 395, 398, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 396, 397, 1, 0, 0, 0, 397, 59, 1, 0, 0, 0, 398, 396, 1, 0, 0, 0, 399, 404, 3, 66, 33, 0, 400, 401, 5, 35, 0, 0, 401, 403, 3, 66, 33, 0, 402, 400, 1, 0, 0, 0, 403, 406, 1, 0, 0, 0, 404, 402, 1, 0, 0, 0, 404, 405, 1, 0, 0, 0, 405, 61, 1, 0, 0, 0, 406, 404, 1, 0, 0, 0, 407, 412, 3, 60, 30, 0, 408, 409, 5, 33, 0, 0, 409, 411, 3, 60, 30, 0, 410, 408, 1, 0, 0, 0, 411, 414, 1, 0, 0, 0, 412, 410, 1, 0, 0, 0, 412, 413, 1, 0, 0, 0, 413, 63, 1, 0, 0, 0, 414, 412, 1, 0, 0, 0, 415, 416, 7, 3, 0, 0, 416, 65, 1, 0, 0, 0, 417, 421, 5, 80, 0, 0, 418, 419, 4, 33, 10, 0, 419, 421, 3, 70, 35, 0, 420, 417, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 421, 67, 1, 0, 0, 0, 422, 465, 5, 44, 0, 0, 423, 424, 3, 104, 52, 0, 424, 425, 5, 67, 0, 0, 425, 465, 1, 0, 0, 0, 426, 465, 3, 102, 51, 0, 427, 465, 3, 104, 52, 0, 428, 465, 3, 98, 49, 0, 429, 465, 3, 70, 35, 0, 430, 465, 3, 106, 53, 0, 431, 432, 5, 65, 0, 0, 432, 437, 3, 100, 50, 0, 433, 434, 5, 33, 0, 0, 434, 436, 3, 100, 50, 0, 435, 433, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 437, 438, 1, 0, 0, 0, 438, 440, 1, 0, 0, 0, 439, 437, 1, 0, 0, 0, 440, 441, 5, 66, 0, 0, 441, 465, 1, 0, 0, 0, 442, 443, 5, 65, 0, 0, 443, 448, 3, 98, 49, 0, 444, 445, 5, 33, 0, 0, 445, 447, 3, 98, 49, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 66, 0, 0, 452, 465, 1, 0, 0, 0, 453, 454, 5, 65, 0, 0, 454, 459, 3, 106, 53, 0, 455, 456, 5, 33, 0, 0, 456, 458, 3, 106, 53, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 66, 0, 0, 463, 465, 1, 0, 0, 0, 464, 422, 1, 0, 0, 0, 464, 423, 1, 0, 0, 0, 464, 426, 1, 0, 0, 0, 464, 427, 1, 0, 0, 0, 464, 428, 1, 0, 0, 0, 464, 429, 1, 0, 0, 0, 464, 430, 1, 0, 0, 0, 464, 431, 1, 0, 0, 0, 464, 442, 1, 0, 0, 0, 464, 453, 1, 0, 0, 0, 465, 69, 1, 0, 0, 0, 466, 469, 5, 47, 0, 0, 467, 469, 5, 64, 0, 0, 468, 466, 1, 0, 0, 0, 468, 467, 1, 0, 0, 0, 469, 71, 1, 0, 0, 0, 470, 474, 3, 64, 32, 0, 471, 472, 4, 36, 11, 0, 472, 474, 3, 70, 35, 0, 473, 470, 1, 0, 0, 0, 473, 471, 1, 0, 0, 0, 474, 73, 1, 0, 0, 0, 475, 476, 5, 9, 0, 0, 476, 477, 5, 26, 0, 0, 477, 75, 1, 0, 0, 0, 478, 479, 5, 14, 0, 0, 479, 484, 3, 78, 39, 0, 480, 481, 5, 33, 0, 0, 481, 483, 3, 78, 39, 0, 482, 480, 1, 0, 0, 0, 483, 486, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 484, 485, 1, 0, 0, 0, 485, 77, 1, 0, 0, 0, 486, 484, 1, 0, 0, 0, 487, 489, 3, 10, 5, 0, 488, 490, 7, 4, 0, 0, 489, 488, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 490, 493, 1, 0, 0, 0, 491, 492, 5, 45, 0, 0, 492, 494, 7, 5, 0, 0, 493, 491, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 79, 1, 0, 0, 0, 495, 496, 5, 8, 0, 0, 496, 497, 3, 62, 31, 0, 497, 81, 1, 0, 0, 0, 498, 499, 5, 2, 0, 0, 499, 500, 3, 62, 31, 0, 500, 83, 1, 0, 0, 0, 501, 502, 5, 11, 0, 0, 502, 507, 3, 86, 43, 0, 503, 504, 5, 33, 0, 0, 504, 506, 3, 86, 43, 0, 505, 503, 1, 0, 0, 0, 506, 509, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 85, 1, 0, 0, 0, 509, 507, 1, 0, 0, 0, 510, 511, 3, 60, 30, 0, 511, 512, 5, 84, 0, 0, 512, 513, 3, 60, 30, 0, 513, 87, 1, 0, 0, 0, 514, 515, 5, 1, 0, 0, 515, 516, 3, 20, 10, 0, 516, 518, 3, 106, 53, 0, 517, 519, 3, 94, 47, 0, 518, 517, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 89, 1, 0, 0, 0, 520, 521, 5, 7, 0, 0, 521, 522, 3, 20, 10, 0, 522, 523, 3, 106, 53, 0, 523, 91, 1, 0, 0, 0, 524, 525, 5, 10, 0, 0, 525, 526, 3, 58, 29, 0, 526, 93, 1, 0, 0, 0, 527, 532, 3, 96, 48, 0, 528, 529, 5, 33, 0, 0, 529, 531, 3, 96, 48, 0, 530, 528, 1, 0, 0, 0, 531, 534, 1, 0, 0, 0, 532, 530, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 95, 1, 0, 0, 0, 534, 532, 1, 0, 0, 0, 535, 536, 3, 64, 32, 0, 536, 537, 5, 31, 0, 0, 537, 538, 3, 68, 34, 0, 538, 97, 1, 0, 0, 0, 539, 540, 7, 6, 0, 0, 540, 99, 1, 0, 0, 0, 541, 544, 3, 102, 51, 0, 542, 544, 3, 104, 52, 0, 543, 541, 1, 0, 0, 0, 543, 542, 1, 0, 0, 0, 544, 101, 1, 0, 0, 0, 545, 547, 7, 0, 0, 0, 546, 545, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 1, 0, 0, 0, 548, 549, 5, 27, 0, 0, 549, 103, 1, 0, 0, 0, 550, 552, 7, 0, 0, 0, 551, 550, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 5, 26, 0, 0, 554, 105, 1, 0, 0, 0, 555, 556, 5, 25, 0, 0, 556, 107, 1, 0, 0, 0, 557, 558, 7, 7, 0, 0, 558, 109, 1, 0, 0, 0, 559, 560, 5, 5, 0, 0, 560, 561, 3, 112, 56, 0, 561, 111, 1, 0, 0, 0, 562, 563, 5, 65, 0, 0, 563, 564, 3, 2, 1, 0, 564, 565, 5, 66, 0, 0, 565, 113, 1, 0, 0, 0, 566, 567, 5, 13, 0, 0, 567, 568, 5, 100, 0, 0, 568, 115, 1, 0, 0, 0, 569, 570, 5, 3, 0, 0, 570, 573, 5, 90, 0, 0, 571, 572, 5, 88, 0, 0, 572, 574, 3, 60, 30, 0, 573, 571, 1, 0, 0, 0, 573, 574, 1, 0, 0, 0, 574, 584, 1, 0, 0, 0, 575, 576, 5, 89, 0, 0, 576, 581, 3, 118, 59, 0, 577, 578, 5, 33, 0, 0, 578, 580, 3, 118, 59, 0, 579, 577, 1, 0, 0, 0, 580, 583, 1, 0, 0, 0, 581, 579, 1, 0, 0, 0, 581, 582, 1, 0, 0, 0, 582, 585, 1, 0, 0, 0, 583, 581, 1, 0, 0, 0, 584, 575, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 117, 1, 0, 0, 0, 586, 587, 3, 60, 30, 0, 587, 588, 5, 31, 0, 0, 588, 590, 1, 0, 0, 0, 589, 586, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 592, 3, 60, 30, 0, 592, 119, 1, 0, 0, 0, 593, 594, 5, 18, 0, 0, 594, 595, 3, 36, 18, 0, 595, 596, 5, 88, 0, 0, 596, 597, 3, 62, 31, 0, 597, 121, 1, 0, 0, 0, 598, 599, 5, 17, 0, 0, 599, 602, 3, 54, 27, 0, 600, 601, 5, 28, 0, 0, 601, 603, 3, 30, 15, 0, 602, 600, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 123, 1, 0, 0, 0, 59, 135, 144, 162, 174, 183, 191, 197, 205, 207, 212, 219, 224, 235, 241, 249, 251, 262, 269, 280, 283, 289, 301, 307, 317, 321, 326, 336, 344, 357, 361, 365, 372, 376, 383, 389, 396, 404, 412, 420, 437, 448, 459, 464, 468, 473, 484, 489, 493, 507, 518, 532, 543, 546, 551, 573, 581, 584, 589, 602]
\ No newline at end of file
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
index 14913849d1b51..9f8197830640c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
@@ -27,13 +27,13 @@ public class EsqlBaseParser extends ParserConfig {
   public static final int
     DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, 
     LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, 
-    WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, 
-    UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, 
-    QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, 
-    ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, 
-    IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, 
-    PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, 
-    GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, 
+    WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, UNKNOWN_CMD=20, 
+    LINE_COMMENT=21, MULTILINE_COMMENT=22, WS=23, PIPE=24, QUOTED_STRING=25, 
+    INTEGER_LITERAL=26, DECIMAL_LITERAL=27, BY=28, AND=29, ASC=30, ASSIGN=31, 
+    CAST_OP=32, COMMA=33, DESC=34, DOT=35, FALSE=36, FIRST=37, IN=38, IS=39, 
+    LAST=40, LIKE=41, LP=42, NOT=43, NULL=44, NULLS=45, OR=46, PARAM=47, RLIKE=48, 
+    RP=49, TRUE=50, EQ=51, CIEQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, 
+    PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, MATCH=63, NAMED_OR_POSITIONAL_PARAM=64, 
     OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, 
     EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, 
     EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, 
@@ -54,36 +54,37 @@ public class EsqlBaseParser extends ParserConfig {
     RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, 
     RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, 
     RULE_matchBooleanExpression = 7, RULE_valueExpression = 8, RULE_operatorExpression = 9, 
-    RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_dataType = 12, 
-    RULE_rowCommand = 13, RULE_fields = 14, RULE_field = 15, RULE_fromCommand = 16, 
-    RULE_indexPattern = 17, RULE_clusterString = 18, RULE_indexString = 19, 
-    RULE_metadata = 20, RULE_metadataOption = 21, RULE_deprecated_metadata = 22, 
-    RULE_metricsCommand = 23, RULE_evalCommand = 24, RULE_statsCommand = 25, 
-    RULE_qualifiedName = 26, RULE_qualifiedNamePattern = 27, RULE_qualifiedNamePatterns = 28, 
-    RULE_identifier = 29, RULE_identifierPattern = 30, RULE_constant = 31, 
-    RULE_parameter = 32, RULE_identifierOrParameter = 33, RULE_limitCommand = 34, 
-    RULE_sortCommand = 35, RULE_orderExpression = 36, RULE_keepCommand = 37, 
-    RULE_dropCommand = 38, RULE_renameCommand = 39, RULE_renameClause = 40, 
-    RULE_dissectCommand = 41, RULE_grokCommand = 42, RULE_mvExpandCommand = 43, 
-    RULE_commandOptions = 44, RULE_commandOption = 45, RULE_booleanValue = 46, 
-    RULE_numericValue = 47, RULE_decimalValue = 48, RULE_integerValue = 49, 
-    RULE_string = 50, RULE_comparisonOperator = 51, RULE_explainCommand = 52, 
-    RULE_subqueryExpression = 53, RULE_showCommand = 54, RULE_enrichCommand = 55, 
-    RULE_enrichWithClause = 56, RULE_lookupCommand = 57, RULE_inlinestatsCommand = 58;
+    RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_functionName = 12, 
+    RULE_dataType = 13, RULE_rowCommand = 14, RULE_fields = 15, RULE_field = 16, 
+    RULE_fromCommand = 17, RULE_indexPattern = 18, RULE_clusterString = 19, 
+    RULE_indexString = 20, RULE_metadata = 21, RULE_metadataOption = 22, RULE_deprecated_metadata = 23, 
+    RULE_metricsCommand = 24, RULE_evalCommand = 25, RULE_statsCommand = 26, 
+    RULE_aggFields = 27, RULE_aggField = 28, RULE_qualifiedName = 29, RULE_qualifiedNamePattern = 30, 
+    RULE_qualifiedNamePatterns = 31, RULE_identifier = 32, RULE_identifierPattern = 33, 
+    RULE_constant = 34, RULE_parameter = 35, RULE_identifierOrParameter = 36, 
+    RULE_limitCommand = 37, RULE_sortCommand = 38, RULE_orderExpression = 39, 
+    RULE_keepCommand = 40, RULE_dropCommand = 41, RULE_renameCommand = 42, 
+    RULE_renameClause = 43, RULE_dissectCommand = 44, RULE_grokCommand = 45, 
+    RULE_mvExpandCommand = 46, RULE_commandOptions = 47, RULE_commandOption = 48, 
+    RULE_booleanValue = 49, RULE_numericValue = 50, RULE_decimalValue = 51, 
+    RULE_integerValue = 52, RULE_string = 53, RULE_comparisonOperator = 54, 
+    RULE_explainCommand = 55, RULE_subqueryExpression = 56, RULE_showCommand = 57, 
+    RULE_enrichCommand = 58, RULE_enrichWithClause = 59, RULE_lookupCommand = 60, 
+    RULE_inlinestatsCommand = 61;
   private static String[] makeRuleNames() {
     return new String[] {
       "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", 
       "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", 
       "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", 
-      "dataType", "rowCommand", "fields", "field", "fromCommand", "indexPattern", 
-      "clusterString", "indexString", "metadata", "metadataOption", "deprecated_metadata", 
-      "metricsCommand", "evalCommand", "statsCommand", "qualifiedName", "qualifiedNamePattern", 
-      "qualifiedNamePatterns", "identifier", "identifierPattern", "constant", 
-      "parameter", "identifierOrParameter", "limitCommand", "sortCommand", 
-      "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", 
-      "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", 
-      "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", 
-      "string", "comparisonOperator", "explainCommand", "subqueryExpression", 
+      "functionName", "dataType", "rowCommand", "fields", "field", "fromCommand", 
+      "indexPattern", "clusterString", "indexString", "metadata", "metadataOption", 
+      "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", 
+      "aggFields", "aggField", "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", 
+      "identifier", "identifierPattern", "constant", "parameter", "identifierOrParameter", 
+      "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", 
+      "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", 
+      "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", 
+      "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", 
       "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", 
       "inlinestatsCommand"
     };
@@ -95,15 +96,15 @@ private static String[] makeLiteralNames() {
       null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", 
       "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", 
       "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, 
-      null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", 
-      "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", 
-      "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", 
-      "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", 
-      "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, 
-      null, null, null, null, null, "'metadata'", null, null, null, null, null, 
-      null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, 
-      null, null, null, null, null, null, null, null, "'info'", null, null, 
-      null, "':'"
+      "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", 
+      "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", 
+      "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", 
+      "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", 
+      "'-'", "'*'", "'/'", "'%'", "'match'", null, null, "']'", null, null, 
+      null, null, null, null, null, null, "'metadata'", null, null, null, null, 
+      null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, 
+      null, null, null, null, null, null, null, null, null, "'info'", null, 
+      null, null, "':'"
     };
   }
   private static final String[] _LITERAL_NAMES = makeLiteralNames();
@@ -111,13 +112,13 @@ private static String[] makeSymbolicNames() {
     return new String[] {
       null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", 
       "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", 
-      "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", 
-      "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", 
-      "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
-      "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", 
-      "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", 
-      "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", 
-      "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
+      "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", 
+      "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", 
+      "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", 
+      "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", 
+      "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", 
+      "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", 
+      "PERCENT", "MATCH", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
       "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", 
       "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", 
       "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", 
@@ -219,9 +220,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(118);
+      setState(124);
       query(0);
-      setState(119);
+      setState(125);
       match(EOF);
       }
     }
@@ -317,11 +318,11 @@ private QueryContext query(int _p) throws RecognitionException {
       _ctx = _localctx;
       _prevctx = _localctx;
 
-      setState(122);
+      setState(128);
       sourceCommand();
       }
       _ctx.stop = _input.LT(-1);
-      setState(129);
+      setState(135);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -332,16 +333,16 @@ private QueryContext query(int _p) throws RecognitionException {
           {
           _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_query);
-          setState(124);
+          setState(130);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(125);
+          setState(131);
           match(PIPE);
-          setState(126);
+          setState(132);
           processingCommand();
           }
           } 
         }
-        setState(131);
+        setState(137);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
@@ -399,43 +400,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException {
     SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState());
     enterRule(_localctx, 4, RULE_sourceCommand);
     try {
-      setState(138);
+      setState(144);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(132);
+        setState(138);
         explainCommand();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(133);
+        setState(139);
         fromCommand();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(134);
+        setState(140);
         rowCommand();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(135);
+        setState(141);
         showCommand();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(136);
+        setState(142);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(137);
+        setState(143);
         metricsCommand();
         }
         break;
@@ -520,108 +521,108 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce
     ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState());
     enterRule(_localctx, 6, RULE_processingCommand);
     try {
-      setState(156);
+      setState(162);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(140);
+        setState(146);
         evalCommand();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(141);
+        setState(147);
         whereCommand();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(142);
+        setState(148);
         keepCommand();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(143);
+        setState(149);
         limitCommand();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(144);
+        setState(150);
         statsCommand();
         }
         break;
       case 6:
         enterOuterAlt(_localctx, 6);
         {
-        setState(145);
+        setState(151);
         sortCommand();
         }
         break;
       case 7:
         enterOuterAlt(_localctx, 7);
         {
-        setState(146);
+        setState(152);
         dropCommand();
         }
         break;
       case 8:
         enterOuterAlt(_localctx, 8);
         {
-        setState(147);
+        setState(153);
         renameCommand();
         }
         break;
       case 9:
         enterOuterAlt(_localctx, 9);
         {
-        setState(148);
+        setState(154);
         dissectCommand();
         }
         break;
       case 10:
         enterOuterAlt(_localctx, 10);
         {
-        setState(149);
+        setState(155);
         grokCommand();
         }
         break;
       case 11:
         enterOuterAlt(_localctx, 11);
         {
-        setState(150);
+        setState(156);
         enrichCommand();
         }
         break;
       case 12:
         enterOuterAlt(_localctx, 12);
         {
-        setState(151);
+        setState(157);
         mvExpandCommand();
         }
         break;
       case 13:
         enterOuterAlt(_localctx, 13);
         {
-        setState(152);
+        setState(158);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(153);
+        setState(159);
         inlinestatsCommand();
         }
         break;
       case 14:
         enterOuterAlt(_localctx, 14);
         {
-        setState(154);
+        setState(160);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(155);
+        setState(161);
         lookupCommand();
         }
         break;
@@ -670,9 +671,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(158);
+      setState(164);
       match(WHERE);
-      setState(159);
+      setState(165);
       booleanExpression(0);
       }
     }
@@ -888,7 +889,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(191);
+      setState(197);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
       case 1:
@@ -897,9 +898,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(162);
+        setState(168);
         match(NOT);
-        setState(163);
+        setState(169);
         booleanExpression(8);
         }
         break;
@@ -908,7 +909,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new BooleanDefaultContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(164);
+        setState(170);
         valueExpression();
         }
         break;
@@ -917,7 +918,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new RegexExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(165);
+        setState(171);
         regexBooleanExpression();
         }
         break;
@@ -926,41 +927,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new LogicalInContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(166);
+        setState(172);
         valueExpression();
-        setState(168);
+        setState(174);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(167);
+          setState(173);
           match(NOT);
           }
         }
 
-        setState(170);
+        setState(176);
         match(IN);
-        setState(171);
+        setState(177);
         match(LP);
-        setState(172);
+        setState(178);
         valueExpression();
-        setState(177);
+        setState(183);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(173);
+          setState(179);
           match(COMMA);
-          setState(174);
+          setState(180);
           valueExpression();
           }
           }
-          setState(179);
+          setState(185);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(180);
+        setState(186);
         match(RP);
         }
         break;
@@ -969,21 +970,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new IsNullContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(182);
+        setState(188);
         valueExpression();
-        setState(183);
+        setState(189);
         match(IS);
-        setState(185);
+        setState(191);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(184);
+          setState(190);
           match(NOT);
           }
         }
 
-        setState(187);
+        setState(193);
         match(NULL);
         }
         break;
@@ -992,15 +993,15 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new MatchExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(189);
+        setState(195);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(190);
+        setState(196);
         matchBooleanExpression();
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(201);
+      setState(207);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1008,7 +1009,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(199);
+          setState(205);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
           case 1:
@@ -1016,11 +1017,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(193);
+            setState(199);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(194);
+            setState(200);
             ((LogicalBinaryContext)_localctx).operator = match(AND);
-            setState(195);
+            setState(201);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(6);
             }
             break;
@@ -1029,18 +1030,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(196);
+            setState(202);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(197);
+            setState(203);
             ((LogicalBinaryContext)_localctx).operator = match(OR);
-            setState(198);
+            setState(204);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(5);
             }
             break;
           }
           } 
         }
-        setState(203);
+        setState(209);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       }
@@ -1095,48 +1096,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog
     enterRule(_localctx, 12, RULE_regexBooleanExpression);
     int _la;
     try {
-      setState(218);
+      setState(224);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(204);
+        setState(210);
         valueExpression();
-        setState(206);
+        setState(212);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(205);
+          setState(211);
           match(NOT);
           }
         }
 
-        setState(208);
+        setState(214);
         ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE);
-        setState(209);
+        setState(215);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(211);
+        setState(217);
         valueExpression();
-        setState(213);
+        setState(219);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(212);
+          setState(218);
           match(NOT);
           }
         }
 
-        setState(215);
+        setState(221);
         ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE);
-        setState(216);
+        setState(222);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
@@ -1159,7 +1160,7 @@ public static class MatchBooleanExpressionContext extends ParserRuleContext {
     public ValueExpressionContext valueExpression() {
       return getRuleContext(ValueExpressionContext.class,0);
     }
-    public TerminalNode DEV_MATCH() { return getToken(EsqlBaseParser.DEV_MATCH, 0); }
+    public TerminalNode MATCH() { return getToken(EsqlBaseParser.MATCH, 0); }
     public StringContext string() {
       return getRuleContext(StringContext.class,0);
     }
@@ -1189,11 +1190,11 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(220);
+      setState(226);
       valueExpression();
-      setState(221);
-      match(DEV_MATCH);
-      setState(222);
+      setState(227);
+      match(MATCH);
+      setState(228);
       ((MatchBooleanExpressionContext)_localctx).queryString = string();
       }
     }
@@ -1277,14 +1278,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
     ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState());
     enterRule(_localctx, 16, RULE_valueExpression);
     try {
-      setState(229);
+      setState(235);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         _localctx = new ValueExpressionDefaultContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(224);
+        setState(230);
         operatorExpression(0);
         }
         break;
@@ -1292,11 +1293,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
         _localctx = new ComparisonContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(225);
+        setState(231);
         ((ComparisonContext)_localctx).left = operatorExpression(0);
-        setState(226);
+        setState(232);
         comparisonOperator();
-        setState(227);
+        setState(233);
         ((ComparisonContext)_localctx).right = operatorExpression(0);
         }
         break;
@@ -1421,7 +1422,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(235);
+      setState(241);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) {
       case 1:
@@ -1430,7 +1431,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(232);
+        setState(238);
         primaryExpression(0);
         }
         break;
@@ -1439,7 +1440,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _localctx = new ArithmeticUnaryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(233);
+        setState(239);
         ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1450,13 +1451,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           _errHandler.reportMatch(this);
           consume();
         }
-        setState(234);
+        setState(240);
         operatorExpression(3);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(245);
+      setState(251);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,15,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1464,7 +1465,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(243);
+          setState(249);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
           case 1:
@@ -1472,12 +1473,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(237);
+            setState(243);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(238);
+            setState(244);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
-            if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -2305843009213693952L) != 0)) ) {
+            if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 8070450532247928832L) != 0)) ) {
               ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this);
             }
             else {
@@ -1485,7 +1486,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(239);
+            setState(245);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3);
             }
             break;
@@ -1494,9 +1495,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(240);
+            setState(246);
             if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-            setState(241);
+            setState(247);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
             if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1507,14 +1508,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(242);
+            setState(248);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2);
             }
             break;
           }
           } 
         }
-        setState(247);
+        setState(253);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,15,_ctx);
       }
@@ -1672,7 +1673,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(256);
+      setState(262);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
       case 1:
@@ -1681,7 +1682,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(249);
+        setState(255);
         constant();
         }
         break;
@@ -1690,7 +1691,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new DereferenceContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(250);
+        setState(256);
         qualifiedName();
         }
         break;
@@ -1699,7 +1700,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new FunctionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(251);
+        setState(257);
         functionExpression();
         }
         break;
@@ -1708,17 +1709,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new ParenthesizedExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(252);
+        setState(258);
         match(LP);
-        setState(253);
+        setState(259);
         booleanExpression(0);
-        setState(254);
+        setState(260);
         match(RP);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(263);
+      setState(269);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1729,16 +1730,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
           {
           _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression);
-          setState(258);
+          setState(264);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(259);
+          setState(265);
           match(CAST_OP);
-          setState(260);
+          setState(266);
           dataType();
           }
           } 
         }
-        setState(265);
+        setState(271);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       }
@@ -1757,8 +1758,8 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
 
   @SuppressWarnings("CheckReturnValue")
   public static class FunctionExpressionContext extends ParserRuleContext {
-    public IdentifierOrParameterContext identifierOrParameter() {
-      return getRuleContext(IdentifierOrParameterContext.class,0);
+    public FunctionNameContext functionName() {
+      return getRuleContext(FunctionNameContext.class,0);
     }
     public TerminalNode LP() { return getToken(EsqlBaseParser.LP, 0); }
     public TerminalNode RP() { return getToken(EsqlBaseParser.RP, 0); }
@@ -1800,37 +1801,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(266);
-      identifierOrParameter();
-      setState(267);
+      setState(272);
+      functionName();
+      setState(273);
       match(LP);
-      setState(277);
+      setState(283);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
       case 1:
         {
-        setState(268);
+        setState(274);
         match(ASTERISK);
         }
         break;
       case 2:
         {
         {
-        setState(269);
+        setState(275);
         booleanExpression(0);
-        setState(274);
+        setState(280);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(270);
+          setState(276);
           match(COMMA);
-          setState(271);
+          setState(277);
           booleanExpression(0);
           }
           }
-          setState(276);
+          setState(282);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
@@ -1838,7 +1839,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
         }
         break;
       }
-      setState(279);
+      setState(285);
       match(RP);
       }
     }
@@ -1853,6 +1854,66 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
     return _localctx;
   }
 
+  @SuppressWarnings("CheckReturnValue")
+  public static class FunctionNameContext extends ParserRuleContext {
+    public TerminalNode MATCH() { return getToken(EsqlBaseParser.MATCH, 0); }
+    public IdentifierOrParameterContext identifierOrParameter() {
+      return getRuleContext(IdentifierOrParameterContext.class,0);
+    }
+    @SuppressWarnings("this-escape")
+    public FunctionNameContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_functionName; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterFunctionName(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitFunctionName(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitFunctionName(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final FunctionNameContext functionName() throws RecognitionException {
+    FunctionNameContext _localctx = new FunctionNameContext(_ctx, getState());
+    enterRule(_localctx, 24, RULE_functionName);
+    try {
+      setState(289);
+      _errHandler.sync(this);
+      switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
+      case 1:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(287);
+        match(MATCH);
+        }
+        break;
+      case 2:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(288);
+        identifierOrParameter();
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   @SuppressWarnings("CheckReturnValue")
   public static class DataTypeContext extends ParserRuleContext {
     @SuppressWarnings("this-escape")
@@ -1891,12 +1952,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DataTypeContext dataType() throws RecognitionException {
     DataTypeContext _localctx = new DataTypeContext(_ctx, getState());
-    enterRule(_localctx, 24, RULE_dataType);
+    enterRule(_localctx, 26, RULE_dataType);
     try {
       _localctx = new ToDataTypeContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(281);
+      setState(291);
       identifier();
       }
     }
@@ -1939,13 +2000,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RowCommandContext rowCommand() throws RecognitionException {
     RowCommandContext _localctx = new RowCommandContext(_ctx, getState());
-    enterRule(_localctx, 26, RULE_rowCommand);
+    enterRule(_localctx, 28, RULE_rowCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(283);
+      setState(293);
       match(ROW);
-      setState(284);
+      setState(294);
       fields();
       }
     }
@@ -1994,30 +2055,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final FieldsContext fields() throws RecognitionException {
     FieldsContext _localctx = new FieldsContext(_ctx, getState());
-    enterRule(_localctx, 28, RULE_fields);
+    enterRule(_localctx, 30, RULE_fields);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(286);
+      setState(296);
       field();
-      setState(291);
+      setState(301);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(287);
+          setState(297);
           match(COMMA);
-          setState(288);
+          setState(298);
           field();
           }
           } 
         }
-        setState(293);
+        setState(303);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       }
       }
     }
@@ -2063,30 +2124,25 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final FieldContext field() throws RecognitionException {
     FieldContext _localctx = new FieldContext(_ctx, getState());
-    enterRule(_localctx, 30, RULE_field);
+    enterRule(_localctx, 32, RULE_field);
     try {
-      setState(299);
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(307);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
       case 1:
-        enterOuterAlt(_localctx, 1);
-        {
-        setState(294);
-        booleanExpression(0);
-        }
-        break;
-      case 2:
-        enterOuterAlt(_localctx, 2);
         {
-        setState(295);
+        setState(304);
         qualifiedName();
-        setState(296);
+        setState(305);
         match(ASSIGN);
-        setState(297);
-        booleanExpression(0);
         }
         break;
       }
+      setState(309);
+      booleanExpression(0);
+      }
     }
     catch (RecognitionException re) {
       _localctx.exception = re;
@@ -2137,39 +2193,39 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final FromCommandContext fromCommand() throws RecognitionException {
     FromCommandContext _localctx = new FromCommandContext(_ctx, getState());
-    enterRule(_localctx, 32, RULE_fromCommand);
+    enterRule(_localctx, 34, RULE_fromCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(301);
+      setState(311);
       match(FROM);
-      setState(302);
+      setState(312);
       indexPattern();
-      setState(307);
+      setState(317);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(303);
+          setState(313);
           match(COMMA);
-          setState(304);
+          setState(314);
           indexPattern();
           }
           } 
         }
-        setState(309);
+        setState(319);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
       }
-      setState(311);
+      setState(321);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         {
-        setState(310);
+        setState(320);
         metadata();
         }
         break;
@@ -2189,13 +2245,13 @@ public final FromCommandContext fromCommand() throws RecognitionException {
 
   @SuppressWarnings("CheckReturnValue")
   public static class IndexPatternContext extends ParserRuleContext {
+    public IndexStringContext indexString() {
+      return getRuleContext(IndexStringContext.class,0);
+    }
     public ClusterStringContext clusterString() {
       return getRuleContext(ClusterStringContext.class,0);
     }
     public TerminalNode COLON() { return getToken(EsqlBaseParser.COLON, 0); }
-    public IndexStringContext indexString() {
-      return getRuleContext(IndexStringContext.class,0);
-    }
     @SuppressWarnings("this-escape")
     public IndexPatternContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
@@ -2218,30 +2274,25 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IndexPatternContext indexPattern() throws RecognitionException {
     IndexPatternContext _localctx = new IndexPatternContext(_ctx, getState());
-    enterRule(_localctx, 34, RULE_indexPattern);
+    enterRule(_localctx, 36, RULE_indexPattern);
     try {
-      setState(318);
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(326);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
-        enterOuterAlt(_localctx, 1);
         {
-        setState(313);
+        setState(323);
         clusterString();
-        setState(314);
+        setState(324);
         match(COLON);
-        setState(315);
-        indexString();
-        }
-        break;
-      case 2:
-        enterOuterAlt(_localctx, 2);
-        {
-        setState(317);
-        indexString();
         }
         break;
       }
+      setState(328);
+      indexString();
+      }
     }
     catch (RecognitionException re) {
       _localctx.exception = re;
@@ -2279,11 +2330,11 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ClusterStringContext clusterString() throws RecognitionException {
     ClusterStringContext _localctx = new ClusterStringContext(_ctx, getState());
-    enterRule(_localctx, 36, RULE_clusterString);
+    enterRule(_localctx, 38, RULE_clusterString);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(320);
+      setState(330);
       match(UNQUOTED_SOURCE);
       }
     }
@@ -2324,12 +2375,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IndexStringContext indexString() throws RecognitionException {
     IndexStringContext _localctx = new IndexStringContext(_ctx, getState());
-    enterRule(_localctx, 38, RULE_indexString);
+    enterRule(_localctx, 40, RULE_indexString);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(322);
+      setState(332);
       _la = _input.LA(1);
       if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) {
       _errHandler.recoverInline(this);
@@ -2382,22 +2433,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetadataContext metadata() throws RecognitionException {
     MetadataContext _localctx = new MetadataContext(_ctx, getState());
-    enterRule(_localctx, 40, RULE_metadata);
+    enterRule(_localctx, 42, RULE_metadata);
     try {
-      setState(326);
+      setState(336);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case METADATA:
         enterOuterAlt(_localctx, 1);
         {
-        setState(324);
+        setState(334);
         metadataOption();
         }
         break;
       case OPENING_BRACKET:
         enterOuterAlt(_localctx, 2);
         {
-        setState(325);
+        setState(335);
         deprecated_metadata();
         }
         break;
@@ -2449,32 +2500,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetadataOptionContext metadataOption() throws RecognitionException {
     MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState());
-    enterRule(_localctx, 42, RULE_metadataOption);
+    enterRule(_localctx, 44, RULE_metadataOption);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(328);
+      setState(338);
       match(METADATA);
-      setState(329);
+      setState(339);
       match(UNQUOTED_SOURCE);
-      setState(334);
+      setState(344);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(330);
+          setState(340);
           match(COMMA);
-          setState(331);
+          setState(341);
           match(UNQUOTED_SOURCE);
           }
           } 
         }
-        setState(336);
+        setState(346);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
       }
       }
     }
@@ -2517,15 +2568,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException {
     Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState());
-    enterRule(_localctx, 44, RULE_deprecated_metadata);
+    enterRule(_localctx, 46, RULE_deprecated_metadata);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(337);
+      setState(347);
       match(OPENING_BRACKET);
-      setState(338);
+      setState(348);
       metadataOption();
-      setState(339);
+      setState(349);
       match(CLOSING_BRACKET);
       }
     }
@@ -2542,7 +2593,7 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition
 
   @SuppressWarnings("CheckReturnValue")
   public static class MetricsCommandContext extends ParserRuleContext {
-    public FieldsContext aggregates;
+    public AggFieldsContext aggregates;
     public FieldsContext grouping;
     public TerminalNode DEV_METRICS() { return getToken(EsqlBaseParser.DEV_METRICS, 0); }
     public List<IndexPatternContext> indexPattern() {
@@ -2556,11 +2607,11 @@ public TerminalNode COMMA(int i) {
       return getToken(EsqlBaseParser.COMMA, i);
     }
     public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); }
-    public List<FieldsContext> fields() {
-      return getRuleContexts(FieldsContext.class);
+    public AggFieldsContext aggFields() {
+      return getRuleContext(AggFieldsContext.class,0);
     }
-    public FieldsContext fields(int i) {
-      return getRuleContext(FieldsContext.class,i);
+    public FieldsContext fields() {
+      return getRuleContext(FieldsContext.class,0);
     }
     @SuppressWarnings("this-escape")
     public MetricsCommandContext(ParserRuleContext parent, int invokingState) {
@@ -2584,51 +2635,51 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetricsCommandContext metricsCommand() throws RecognitionException {
     MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState());
-    enterRule(_localctx, 46, RULE_metricsCommand);
+    enterRule(_localctx, 48, RULE_metricsCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(341);
+      setState(351);
       match(DEV_METRICS);
-      setState(342);
+      setState(352);
       indexPattern();
-      setState(347);
+      setState(357);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(343);
+          setState(353);
           match(COMMA);
-          setState(344);
+          setState(354);
           indexPattern();
           }
           } 
         }
-        setState(349);
+        setState(359);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
       }
-      setState(351);
+      setState(361);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(350);
-        ((MetricsCommandContext)_localctx).aggregates = fields();
+        setState(360);
+        ((MetricsCommandContext)_localctx).aggregates = aggFields();
         }
         break;
       }
-      setState(355);
+      setState(365);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(353);
+        setState(363);
         match(BY);
-        setState(354);
+        setState(364);
         ((MetricsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2674,13 +2725,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EvalCommandContext evalCommand() throws RecognitionException {
     EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState());
-    enterRule(_localctx, 48, RULE_evalCommand);
+    enterRule(_localctx, 50, RULE_evalCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(357);
+      setState(367);
       match(EVAL);
-      setState(358);
+      setState(368);
       fields();
       }
     }
@@ -2697,15 +2748,15 @@ public final EvalCommandContext evalCommand() throws RecognitionException {
 
   @SuppressWarnings("CheckReturnValue")
   public static class StatsCommandContext extends ParserRuleContext {
-    public FieldsContext stats;
+    public AggFieldsContext stats;
     public FieldsContext grouping;
     public TerminalNode STATS() { return getToken(EsqlBaseParser.STATS, 0); }
     public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); }
-    public List<FieldsContext> fields() {
-      return getRuleContexts(FieldsContext.class);
+    public AggFieldsContext aggFields() {
+      return getRuleContext(AggFieldsContext.class,0);
     }
-    public FieldsContext fields(int i) {
-      return getRuleContext(FieldsContext.class,i);
+    public FieldsContext fields() {
+      return getRuleContext(FieldsContext.class,0);
     }
     @SuppressWarnings("this-escape")
     public StatsCommandContext(ParserRuleContext parent, int invokingState) {
@@ -2729,30 +2780,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final StatsCommandContext statsCommand() throws RecognitionException {
     StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState());
-    enterRule(_localctx, 50, RULE_statsCommand);
+    enterRule(_localctx, 52, RULE_statsCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(360);
+      setState(370);
       match(STATS);
-      setState(362);
+      setState(372);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         {
-        setState(361);
-        ((StatsCommandContext)_localctx).stats = fields();
+        setState(371);
+        ((StatsCommandContext)_localctx).stats = aggFields();
         }
         break;
       }
-      setState(366);
+      setState(376);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
       case 1:
         {
-        setState(364);
+        setState(374);
         match(BY);
-        setState(365);
+        setState(375);
         ((StatsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2770,6 +2821,140 @@ public final StatsCommandContext statsCommand() throws RecognitionException {
     return _localctx;
   }
 
+  @SuppressWarnings("CheckReturnValue")
+  public static class AggFieldsContext extends ParserRuleContext {
+    public List<AggFieldContext> aggField() {
+      return getRuleContexts(AggFieldContext.class);
+    }
+    public AggFieldContext aggField(int i) {
+      return getRuleContext(AggFieldContext.class,i);
+    }
+    public List<TerminalNode> COMMA() { return getTokens(EsqlBaseParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(EsqlBaseParser.COMMA, i);
+    }
+    @SuppressWarnings("this-escape")
+    public AggFieldsContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_aggFields; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterAggFields(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitAggFields(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitAggFields(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final AggFieldsContext aggFields() throws RecognitionException {
+    AggFieldsContext _localctx = new AggFieldsContext(_ctx, getState());
+    enterRule(_localctx, 54, RULE_aggFields);
+    try {
+      int _alt;
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(378);
+      aggField();
+      setState(383);
+      _errHandler.sync(this);
+      _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+        if ( _alt==1 ) {
+          {
+          {
+          setState(379);
+          match(COMMA);
+          setState(380);
+          aggField();
+          }
+          } 
+        }
+        setState(385);
+        _errHandler.sync(this);
+        _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+      }
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  @SuppressWarnings("CheckReturnValue")
+  public static class AggFieldContext extends ParserRuleContext {
+    public FieldContext field() {
+      return getRuleContext(FieldContext.class,0);
+    }
+    public TerminalNode WHERE() { return getToken(EsqlBaseParser.WHERE, 0); }
+    public BooleanExpressionContext booleanExpression() {
+      return getRuleContext(BooleanExpressionContext.class,0);
+    }
+    @SuppressWarnings("this-escape")
+    public AggFieldContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_aggField; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterAggField(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitAggField(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitAggField(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final AggFieldContext aggField() throws RecognitionException {
+    AggFieldContext _localctx = new AggFieldContext(_ctx, getState());
+    enterRule(_localctx, 56, RULE_aggField);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(386);
+      field();
+      setState(389);
+      _errHandler.sync(this);
+      switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
+      case 1:
+        {
+        setState(387);
+        match(WHERE);
+        setState(388);
+        booleanExpression(0);
+        }
+        break;
+      }
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   @SuppressWarnings("CheckReturnValue")
   public static class QualifiedNameContext extends ParserRuleContext {
     public List<IdentifierOrParameterContext> identifierOrParameter() {
@@ -2804,30 +2989,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final QualifiedNameContext qualifiedName() throws RecognitionException {
     QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState());
-    enterRule(_localctx, 52, RULE_qualifiedName);
+    enterRule(_localctx, 58, RULE_qualifiedName);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(368);
+      setState(391);
       identifierOrParameter();
-      setState(373);
+      setState(396);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(369);
+          setState(392);
           match(DOT);
-          setState(370);
+          setState(393);
           identifierOrParameter();
           }
           } 
         }
-        setState(375);
+        setState(398);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
       }
       }
     }
@@ -2876,30 +3061,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException {
     QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState());
-    enterRule(_localctx, 54, RULE_qualifiedNamePattern);
+    enterRule(_localctx, 60, RULE_qualifiedNamePattern);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(376);
+      setState(399);
       identifierPattern();
-      setState(381);
+      setState(404);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,36,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(377);
+          setState(400);
           match(DOT);
-          setState(378);
+          setState(401);
           identifierPattern();
           }
           } 
         }
-        setState(383);
+        setState(406);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,36,_ctx);
       }
       }
     }
@@ -2948,30 +3133,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException {
     QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState());
-    enterRule(_localctx, 56, RULE_qualifiedNamePatterns);
+    enterRule(_localctx, 62, RULE_qualifiedNamePatterns);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(384);
+      setState(407);
       qualifiedNamePattern();
-      setState(389);
+      setState(412);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,37,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(385);
+          setState(408);
           match(COMMA);
-          setState(386);
+          setState(409);
           qualifiedNamePattern();
           }
           } 
         }
-        setState(391);
+        setState(414);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,37,_ctx);
       }
       }
     }
@@ -3012,12 +3197,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierContext identifier() throws RecognitionException {
     IdentifierContext _localctx = new IdentifierContext(_ctx, getState());
-    enterRule(_localctx, 58, RULE_identifier);
+    enterRule(_localctx, 64, RULE_identifier);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(392);
+      setState(415);
       _la = _input.LA(1);
       if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) {
       _errHandler.recoverInline(this);
@@ -3068,28 +3253,27 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierPatternContext identifierPattern() throws RecognitionException {
     IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState());
-    enterRule(_localctx, 60, RULE_identifierPattern);
+    enterRule(_localctx, 66, RULE_identifierPattern);
     try {
-      setState(396);
+      setState(420);
       _errHandler.sync(this);
-      switch (_input.LA(1)) {
-      case ID_PATTERN:
+      switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
+      case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(394);
+        setState(417);
         match(ID_PATTERN);
         }
         break;
-      case PARAM:
-      case NAMED_OR_POSITIONAL_PARAM:
+      case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(395);
+        setState(418);
+        if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
+        setState(419);
         parameter();
         }
         break;
-      default:
-        throw new NoViableAltException(this);
       }
     }
     catch (RecognitionException re) {
@@ -3356,17 +3540,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ConstantContext constant() throws RecognitionException {
     ConstantContext _localctx = new ConstantContext(_ctx, getState());
-    enterRule(_localctx, 62, RULE_constant);
+    enterRule(_localctx, 68, RULE_constant);
     int _la;
     try {
-      setState(440);
+      setState(464);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) {
       case 1:
         _localctx = new NullLiteralContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(398);
+        setState(422);
         match(NULL);
         }
         break;
@@ -3374,9 +3558,9 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new QualifiedIntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(399);
+        setState(423);
         integerValue();
-        setState(400);
+        setState(424);
         match(UNQUOTED_IDENTIFIER);
         }
         break;
@@ -3384,7 +3568,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new DecimalLiteralContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(402);
+        setState(426);
         decimalValue();
         }
         break;
@@ -3392,7 +3576,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new IntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(403);
+        setState(427);
         integerValue();
         }
         break;
@@ -3400,7 +3584,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanLiteralContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(404);
+        setState(428);
         booleanValue();
         }
         break;
@@ -3408,7 +3592,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new InputParameterContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(405);
+        setState(429);
         parameter();
         }
         break;
@@ -3416,7 +3600,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringLiteralContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(406);
+        setState(430);
         string();
         }
         break;
@@ -3424,27 +3608,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new NumericArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(407);
+        setState(431);
         match(OPENING_BRACKET);
-        setState(408);
+        setState(432);
         numericValue();
-        setState(413);
+        setState(437);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(409);
+          setState(433);
           match(COMMA);
-          setState(410);
+          setState(434);
           numericValue();
           }
           }
-          setState(415);
+          setState(439);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(416);
+        setState(440);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3452,27 +3636,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(418);
+        setState(442);
         match(OPENING_BRACKET);
-        setState(419);
+        setState(443);
         booleanValue();
-        setState(424);
+        setState(448);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(420);
+          setState(444);
           match(COMMA);
-          setState(421);
+          setState(445);
           booleanValue();
           }
           }
-          setState(426);
+          setState(450);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(427);
+        setState(451);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3480,27 +3664,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(429);
+        setState(453);
         match(OPENING_BRACKET);
-        setState(430);
+        setState(454);
         string();
-        setState(435);
+        setState(459);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(431);
+          setState(455);
           match(COMMA);
-          setState(432);
+          setState(456);
           string();
           }
           }
-          setState(437);
+          setState(461);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(438);
+        setState(462);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3572,16 +3756,16 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ParameterContext parameter() throws RecognitionException {
     ParameterContext _localctx = new ParameterContext(_ctx, getState());
-    enterRule(_localctx, 64, RULE_parameter);
+    enterRule(_localctx, 70, RULE_parameter);
     try {
-      setState(444);
+      setState(468);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case PARAM:
         _localctx = new InputParamContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(442);
+        setState(466);
         match(PARAM);
         }
         break;
@@ -3589,7 +3773,7 @@ public final ParameterContext parameter() throws RecognitionException {
         _localctx = new InputNamedOrPositionalParamContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(443);
+        setState(467);
         match(NAMED_OR_POSITIONAL_PARAM);
         }
         break;
@@ -3638,29 +3822,27 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierOrParameterContext identifierOrParameter() throws RecognitionException {
     IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState());
-    enterRule(_localctx, 66, RULE_identifierOrParameter);
+    enterRule(_localctx, 72, RULE_identifierOrParameter);
     try {
-      setState(448);
+      setState(473);
       _errHandler.sync(this);
-      switch (_input.LA(1)) {
-      case UNQUOTED_IDENTIFIER:
-      case QUOTED_IDENTIFIER:
+      switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) {
+      case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(446);
+        setState(470);
         identifier();
         }
         break;
-      case PARAM:
-      case NAMED_OR_POSITIONAL_PARAM:
+      case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(447);
+        setState(471);
+        if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
+        setState(472);
         parameter();
         }
         break;
-      default:
-        throw new NoViableAltException(this);
       }
     }
     catch (RecognitionException re) {
@@ -3700,13 +3882,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final LimitCommandContext limitCommand() throws RecognitionException {
     LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState());
-    enterRule(_localctx, 68, RULE_limitCommand);
+    enterRule(_localctx, 74, RULE_limitCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(450);
+      setState(475);
       match(LIMIT);
-      setState(451);
+      setState(476);
       match(INTEGER_LITERAL);
       }
     }
@@ -3756,32 +3938,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final SortCommandContext sortCommand() throws RecognitionException {
     SortCommandContext _localctx = new SortCommandContext(_ctx, getState());
-    enterRule(_localctx, 70, RULE_sortCommand);
+    enterRule(_localctx, 76, RULE_sortCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(453);
+      setState(478);
       match(SORT);
-      setState(454);
+      setState(479);
       orderExpression();
-      setState(459);
+      setState(484);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,42,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(455);
+          setState(480);
           match(COMMA);
-          setState(456);
+          setState(481);
           orderExpression();
           }
           } 
         }
-        setState(461);
+        setState(486);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,42,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
       }
       }
     }
@@ -3830,19 +4012,19 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final OrderExpressionContext orderExpression() throws RecognitionException {
     OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState());
-    enterRule(_localctx, 72, RULE_orderExpression);
+    enterRule(_localctx, 78, RULE_orderExpression);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(462);
+      setState(487);
       booleanExpression(0);
-      setState(464);
+      setState(489);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) {
       case 1:
         {
-        setState(463);
+        setState(488);
         ((OrderExpressionContext)_localctx).ordering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==ASC || _la==DESC) ) {
@@ -3856,14 +4038,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio
         }
         break;
       }
-      setState(468);
+      setState(493);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) {
       case 1:
         {
-        setState(466);
+        setState(491);
         match(NULLS);
-        setState(467);
+        setState(492);
         ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==FIRST || _la==LAST) ) {
@@ -3918,13 +4100,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final KeepCommandContext keepCommand() throws RecognitionException {
     KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState());
-    enterRule(_localctx, 74, RULE_keepCommand);
+    enterRule(_localctx, 80, RULE_keepCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(470);
+      setState(495);
       match(KEEP);
-      setState(471);
+      setState(496);
       qualifiedNamePatterns();
       }
     }
@@ -3967,13 +4149,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DropCommandContext dropCommand() throws RecognitionException {
     DropCommandContext _localctx = new DropCommandContext(_ctx, getState());
-    enterRule(_localctx, 76, RULE_dropCommand);
+    enterRule(_localctx, 82, RULE_dropCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(473);
+      setState(498);
       match(DROP);
-      setState(474);
+      setState(499);
       qualifiedNamePatterns();
       }
     }
@@ -4023,32 +4205,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RenameCommandContext renameCommand() throws RecognitionException {
     RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState());
-    enterRule(_localctx, 78, RULE_renameCommand);
+    enterRule(_localctx, 84, RULE_renameCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(476);
+      setState(501);
       match(RENAME);
-      setState(477);
+      setState(502);
       renameClause();
-      setState(482);
+      setState(507);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,48,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(478);
+          setState(503);
           match(COMMA);
-          setState(479);
+          setState(504);
           renameClause();
           }
           } 
         }
-        setState(484);
+        setState(509);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,48,_ctx);
       }
       }
     }
@@ -4096,15 +4278,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RenameClauseContext renameClause() throws RecognitionException {
     RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState());
-    enterRule(_localctx, 80, RULE_renameClause);
+    enterRule(_localctx, 86, RULE_renameClause);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(485);
+      setState(510);
       ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern();
-      setState(486);
+      setState(511);
       match(AS);
-      setState(487);
+      setState(512);
       ((RenameClauseContext)_localctx).newName = qualifiedNamePattern();
       }
     }
@@ -4153,22 +4335,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DissectCommandContext dissectCommand() throws RecognitionException {
     DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState());
-    enterRule(_localctx, 82, RULE_dissectCommand);
+    enterRule(_localctx, 88, RULE_dissectCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(489);
+      setState(514);
       match(DISSECT);
-      setState(490);
+      setState(515);
       primaryExpression(0);
-      setState(491);
+      setState(516);
       string();
-      setState(493);
+      setState(518);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) {
       case 1:
         {
-        setState(492);
+        setState(517);
         commandOptions();
         }
         break;
@@ -4217,15 +4399,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final GrokCommandContext grokCommand() throws RecognitionException {
     GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState());
-    enterRule(_localctx, 84, RULE_grokCommand);
+    enterRule(_localctx, 90, RULE_grokCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(495);
+      setState(520);
       match(GROK);
-      setState(496);
+      setState(521);
       primaryExpression(0);
-      setState(497);
+      setState(522);
       string();
       }
     }
@@ -4268,13 +4450,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MvExpandCommandContext mvExpandCommand() throws RecognitionException {
     MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState());
-    enterRule(_localctx, 86, RULE_mvExpandCommand);
+    enterRule(_localctx, 92, RULE_mvExpandCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(499);
+      setState(524);
       match(MV_EXPAND);
-      setState(500);
+      setState(525);
       qualifiedName();
       }
     }
@@ -4323,30 +4505,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final CommandOptionsContext commandOptions() throws RecognitionException {
     CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState());
-    enterRule(_localctx, 88, RULE_commandOptions);
+    enterRule(_localctx, 94, RULE_commandOptions);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(502);
+      setState(527);
       commandOption();
-      setState(507);
+      setState(532);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,47,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(503);
+          setState(528);
           match(COMMA);
-          setState(504);
+          setState(529);
           commandOption();
           }
           } 
         }
-        setState(509);
+        setState(534);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,47,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
       }
       }
     }
@@ -4392,15 +4574,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final CommandOptionContext commandOption() throws RecognitionException {
     CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState());
-    enterRule(_localctx, 90, RULE_commandOption);
+    enterRule(_localctx, 96, RULE_commandOption);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(510);
+      setState(535);
       identifier();
-      setState(511);
+      setState(536);
       match(ASSIGN);
-      setState(512);
+      setState(537);
       constant();
       }
     }
@@ -4441,12 +4623,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final BooleanValueContext booleanValue() throws RecognitionException {
     BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState());
-    enterRule(_localctx, 92, RULE_booleanValue);
+    enterRule(_localctx, 98, RULE_booleanValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(514);
+      setState(539);
       _la = _input.LA(1);
       if ( !(_la==FALSE || _la==TRUE) ) {
       _errHandler.recoverInline(this);
@@ -4499,22 +4681,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final NumericValueContext numericValue() throws RecognitionException {
     NumericValueContext _localctx = new NumericValueContext(_ctx, getState());
-    enterRule(_localctx, 94, RULE_numericValue);
+    enterRule(_localctx, 100, RULE_numericValue);
     try {
-      setState(518);
+      setState(543);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(516);
+        setState(541);
         decimalValue();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(517);
+        setState(542);
         integerValue();
         }
         break;
@@ -4558,17 +4740,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DecimalValueContext decimalValue() throws RecognitionException {
     DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState());
-    enterRule(_localctx, 96, RULE_decimalValue);
+    enterRule(_localctx, 102, RULE_decimalValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(521);
+      setState(546);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(520);
+        setState(545);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4581,7 +4763,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException {
         }
       }
 
-      setState(523);
+      setState(548);
       match(DECIMAL_LITERAL);
       }
     }
@@ -4623,17 +4805,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IntegerValueContext integerValue() throws RecognitionException {
     IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState());
-    enterRule(_localctx, 98, RULE_integerValue);
+    enterRule(_localctx, 104, RULE_integerValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(526);
+      setState(551);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(525);
+        setState(550);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4646,7 +4828,7 @@ public final IntegerValueContext integerValue() throws RecognitionException {
         }
       }
 
-      setState(528);
+      setState(553);
       match(INTEGER_LITERAL);
       }
     }
@@ -4686,11 +4868,11 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final StringContext string() throws RecognitionException {
     StringContext _localctx = new StringContext(_ctx, getState());
-    enterRule(_localctx, 100, RULE_string);
+    enterRule(_localctx, 106, RULE_string);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(530);
+      setState(555);
       match(QUOTED_STRING);
       }
     }
@@ -4735,14 +4917,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ComparisonOperatorContext comparisonOperator() throws RecognitionException {
     ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState());
-    enterRule(_localctx, 102, RULE_comparisonOperator);
+    enterRule(_localctx, 108, RULE_comparisonOperator);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(532);
+      setState(557);
       _la = _input.LA(1);
-      if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 562949953421312000L) != 0)) ) {
+      if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 281474976710656000L) != 0)) ) {
       _errHandler.recoverInline(this);
       }
       else {
@@ -4791,13 +4973,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ExplainCommandContext explainCommand() throws RecognitionException {
     ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState());
-    enterRule(_localctx, 104, RULE_explainCommand);
+    enterRule(_localctx, 110, RULE_explainCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(534);
+      setState(559);
       match(EXPLAIN);
-      setState(535);
+      setState(560);
       subqueryExpression();
       }
     }
@@ -4841,15 +5023,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final SubqueryExpressionContext subqueryExpression() throws RecognitionException {
     SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState());
-    enterRule(_localctx, 106, RULE_subqueryExpression);
+    enterRule(_localctx, 112, RULE_subqueryExpression);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(537);
+      setState(562);
       match(OPENING_BRACKET);
-      setState(538);
+      setState(563);
       query(0);
-      setState(539);
+      setState(564);
       match(CLOSING_BRACKET);
       }
     }
@@ -4901,14 +5083,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ShowCommandContext showCommand() throws RecognitionException {
     ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState());
-    enterRule(_localctx, 108, RULE_showCommand);
+    enterRule(_localctx, 114, RULE_showCommand);
     try {
       _localctx = new ShowInfoContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(541);
+      setState(566);
       match(SHOW);
-      setState(542);
+      setState(567);
       match(INFO);
       }
     }
@@ -4966,53 +5148,53 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EnrichCommandContext enrichCommand() throws RecognitionException {
     EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState());
-    enterRule(_localctx, 110, RULE_enrichCommand);
+    enterRule(_localctx, 116, RULE_enrichCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(544);
+      setState(569);
       match(ENRICH);
-      setState(545);
+      setState(570);
       ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME);
-      setState(548);
+      setState(573);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) {
       case 1:
         {
-        setState(546);
+        setState(571);
         match(ON);
-        setState(547);
+        setState(572);
         ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern();
         }
         break;
       }
-      setState(559);
+      setState(584);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) {
       case 1:
         {
-        setState(550);
+        setState(575);
         match(WITH);
-        setState(551);
+        setState(576);
         enrichWithClause();
-        setState(556);
+        setState(581);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,52,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,55,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(552);
+            setState(577);
             match(COMMA);
-            setState(553);
+            setState(578);
             enrichWithClause();
             }
             } 
           }
-          setState(558);
+          setState(583);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,52,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,55,_ctx);
         }
         }
         break;
@@ -5063,23 +5245,23 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EnrichWithClauseContext enrichWithClause() throws RecognitionException {
     EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState());
-    enterRule(_localctx, 112, RULE_enrichWithClause);
+    enterRule(_localctx, 118, RULE_enrichWithClause);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(564);
+      setState(589);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) {
       case 1:
         {
-        setState(561);
+        setState(586);
         ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern();
-        setState(562);
+        setState(587);
         match(ASSIGN);
         }
         break;
       }
-      setState(566);
+      setState(591);
       ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern();
       }
     }
@@ -5128,17 +5310,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final LookupCommandContext lookupCommand() throws RecognitionException {
     LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState());
-    enterRule(_localctx, 114, RULE_lookupCommand);
+    enterRule(_localctx, 120, RULE_lookupCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(568);
+      setState(593);
       match(DEV_LOOKUP);
-      setState(569);
+      setState(594);
       ((LookupCommandContext)_localctx).tableName = indexPattern();
-      setState(570);
+      setState(595);
       match(ON);
-      setState(571);
+      setState(596);
       ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns();
       }
     }
@@ -5155,16 +5337,16 @@ public final LookupCommandContext lookupCommand() throws RecognitionException {
 
   @SuppressWarnings("CheckReturnValue")
   public static class InlinestatsCommandContext extends ParserRuleContext {
-    public FieldsContext stats;
+    public AggFieldsContext stats;
     public FieldsContext grouping;
     public TerminalNode DEV_INLINESTATS() { return getToken(EsqlBaseParser.DEV_INLINESTATS, 0); }
-    public List<FieldsContext> fields() {
-      return getRuleContexts(FieldsContext.class);
-    }
-    public FieldsContext fields(int i) {
-      return getRuleContext(FieldsContext.class,i);
+    public AggFieldsContext aggFields() {
+      return getRuleContext(AggFieldsContext.class,0);
     }
     public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); }
+    public FieldsContext fields() {
+      return getRuleContext(FieldsContext.class,0);
+    }
     @SuppressWarnings("this-escape")
     public InlinestatsCommandContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
@@ -5187,22 +5369,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException {
     InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState());
-    enterRule(_localctx, 116, RULE_inlinestatsCommand);
+    enterRule(_localctx, 122, RULE_inlinestatsCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(573);
+      setState(598);
       match(DEV_INLINESTATS);
-      setState(574);
-      ((InlinestatsCommandContext)_localctx).stats = fields();
-      setState(577);
+      setState(599);
+      ((InlinestatsCommandContext)_localctx).stats = aggFields();
+      setState(602);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) {
       case 1:
         {
-        setState(575);
+        setState(600);
         match(BY);
-        setState(576);
+        setState(601);
         ((InlinestatsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -5234,6 +5416,10 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
       return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex);
     case 10:
       return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex);
+    case 33:
+      return identifierPattern_sempred((IdentifierPatternContext)_localctx, predIndex);
+    case 36:
+      return identifierOrParameter_sempred((IdentifierOrParameterContext)_localctx, predIndex);
     }
     return true;
   }
@@ -5287,9 +5473,23 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in
     }
     return true;
   }
+  private boolean identifierPattern_sempred(IdentifierPatternContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 10:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 11:
+      return this.isDevVersion();
+    }
+    return true;
+  }
 
   public static final String _serializedATN =
-    "\u0004\u0001x\u0244\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
+    "\u0004\u0001x\u025d\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
     "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+
     "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+
     "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+
@@ -5304,360 +5504,373 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in
     "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+
     "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+
     "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+
-    "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0001\u0000\u0001\u0000"+
-    "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
-    "\u0001\u0001\u0005\u0001\u0080\b\u0001\n\u0001\f\u0001\u0083\t\u0001\u0001"+
-    "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003"+
-    "\u0002\u008b\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
+    "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002"+
+    "<\u0007<\u0002=\u0007=\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001"+
+    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001"+
+    "\u0086\b\u0001\n\u0001\f\u0001\u0089\t\u0001\u0001\u0002\u0001\u0002\u0001"+
+    "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0091\b\u0002\u0001"+
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
     "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
-    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003"+
-    "\u0003\u009d\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+
-    "\u0005\u00a9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
-    "\u0005\u0005\u0005\u00b0\b\u0005\n\u0005\f\u0005\u00b3\t\u0005\u0001\u0005"+
-    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00ba\b\u0005"+
-    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00c0\b\u0005"+
-    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
-    "\u0005\u0005\u00c8\b\u0005\n\u0005\f\u0005\u00cb\t\u0005\u0001\u0006\u0001"+
-    "\u0006\u0003\u0006\u00cf\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+
-    "\u0006\u0001\u0006\u0003\u0006\u00d6\b\u0006\u0001\u0006\u0001\u0006\u0001"+
-    "\u0006\u0003\u0006\u00db\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+
-    "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00e6\b\b\u0001"+
-    "\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ec\b\t\u0001\t\u0001\t\u0001\t\u0001"+
-    "\t\u0001\t\u0001\t\u0005\t\u00f4\b\t\n\t\f\t\u00f7\t\t\u0001\n\u0001\n"+
-    "\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0101\b\n\u0001"+
-    "\n\u0001\n\u0001\n\u0005\n\u0106\b\n\n\n\f\n\u0109\t\n\u0001\u000b\u0001"+
-    "\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0111"+
-    "\b\u000b\n\u000b\f\u000b\u0114\t\u000b\u0003\u000b\u0116\b\u000b\u0001"+
-    "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\u000e"+
-    "\u0001\u000e\u0001\u000e\u0005\u000e\u0122\b\u000e\n\u000e\f\u000e\u0125"+
-    "\t\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0003"+
-    "\u000f\u012c\b\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005"+
-    "\u0010\u0132\b\u0010\n\u0010\f\u0010\u0135\t\u0010\u0001\u0010\u0003\u0010"+
-    "\u0138\b\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+
-    "\u0003\u0011\u013f\b\u0011\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+
-    "\u0001\u0014\u0001\u0014\u0003\u0014\u0147\b\u0014\u0001\u0015\u0001\u0015"+
-    "\u0001\u0015\u0001\u0015\u0005\u0015\u014d\b\u0015\n\u0015\f\u0015\u0150"+
-    "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+
-    "\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u015a\b\u0017\n\u0017\f\u0017"+
-    "\u015d\t\u0017\u0001\u0017\u0003\u0017\u0160\b\u0017\u0001\u0017\u0001"+
-    "\u0017\u0003\u0017\u0164\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001"+
-    "\u0019\u0001\u0019\u0003\u0019\u016b\b\u0019\u0001\u0019\u0001\u0019\u0003"+
-    "\u0019\u016f\b\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u0174"+
-    "\b\u001a\n\u001a\f\u001a\u0177\t\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+
-    "\u0005\u001b\u017c\b\u001b\n\u001b\f\u001b\u017f\t\u001b\u0001\u001c\u0001"+
-    "\u001c\u0001\u001c\u0005\u001c\u0184\b\u001c\n\u001c\f\u001c\u0187\t\u001c"+
-    "\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0003\u001e\u018d\b\u001e"+
-    "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+
-    "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+
-    "\u0001\u001f\u0005\u001f\u019c\b\u001f\n\u001f\f\u001f\u019f\t\u001f\u0001"+
-    "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005"+
-    "\u001f\u01a7\b\u001f\n\u001f\f\u001f\u01aa\t\u001f\u0001\u001f\u0001\u001f"+
-    "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01b2\b\u001f"+
-    "\n\u001f\f\u001f\u01b5\t\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01b9"+
-    "\b\u001f\u0001 \u0001 \u0003 \u01bd\b \u0001!\u0001!\u0003!\u01c1\b!\u0001"+
-    "\"\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0001#\u0005#\u01ca\b#\n#\f#\u01cd"+
-    "\t#\u0001$\u0001$\u0003$\u01d1\b$\u0001$\u0001$\u0003$\u01d5\b$\u0001"+
-    "%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0005"+
-    "\'\u01e1\b\'\n\'\f\'\u01e4\t\'\u0001(\u0001(\u0001(\u0001(\u0001)\u0001"+
-    ")\u0001)\u0001)\u0003)\u01ee\b)\u0001*\u0001*\u0001*\u0001*\u0001+\u0001"+
-    "+\u0001+\u0001,\u0001,\u0001,\u0005,\u01fa\b,\n,\f,\u01fd\t,\u0001-\u0001"+
-    "-\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0003/\u0207\b/\u00010\u0003"+
-    "0\u020a\b0\u00010\u00010\u00011\u00031\u020f\b1\u00011\u00011\u00012\u0001"+
-    "2\u00013\u00013\u00014\u00014\u00014\u00015\u00015\u00015\u00015\u0001"+
-    "6\u00016\u00016\u00017\u00017\u00017\u00017\u00037\u0225\b7\u00017\u0001"+
-    "7\u00017\u00017\u00057\u022b\b7\n7\f7\u022e\t7\u00037\u0230\b7\u00018"+
-    "\u00018\u00018\u00038\u0235\b8\u00018\u00018\u00019\u00019\u00019\u0001"+
-    "9\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u0242\b:\u0001:\u0000\u0004"+
-    "\u0002\n\u0012\u0014;\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012"+
-    "\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\"+
-    "^`bdfhjlnprt\u0000\b\u0001\u0000;<\u0001\u0000=?\u0002\u0000\u001a\u001a"+
-    "LL\u0001\u0000CD\u0002\u0000\u001f\u001f##\u0002\u0000&&))\u0002\u0000"+
-    "%%33\u0002\u0000446:\u025e\u0000v\u0001\u0000\u0000\u0000\u0002y\u0001"+
-    "\u0000\u0000\u0000\u0004\u008a\u0001\u0000\u0000\u0000\u0006\u009c\u0001"+
-    "\u0000\u0000\u0000\b\u009e\u0001\u0000\u0000\u0000\n\u00bf\u0001\u0000"+
-    "\u0000\u0000\f\u00da\u0001\u0000\u0000\u0000\u000e\u00dc\u0001\u0000\u0000"+
-    "\u0000\u0010\u00e5\u0001\u0000\u0000\u0000\u0012\u00eb\u0001\u0000\u0000"+
-    "\u0000\u0014\u0100\u0001\u0000\u0000\u0000\u0016\u010a\u0001\u0000\u0000"+
-    "\u0000\u0018\u0119\u0001\u0000\u0000\u0000\u001a\u011b\u0001\u0000\u0000"+
-    "\u0000\u001c\u011e\u0001\u0000\u0000\u0000\u001e\u012b\u0001\u0000\u0000"+
-    "\u0000 \u012d\u0001\u0000\u0000\u0000\"\u013e\u0001\u0000\u0000\u0000"+
-    "$\u0140\u0001\u0000\u0000\u0000&\u0142\u0001\u0000\u0000\u0000(\u0146"+
-    "\u0001\u0000\u0000\u0000*\u0148\u0001\u0000\u0000\u0000,\u0151\u0001\u0000"+
-    "\u0000\u0000.\u0155\u0001\u0000\u0000\u00000\u0165\u0001\u0000\u0000\u0000"+
-    "2\u0168\u0001\u0000\u0000\u00004\u0170\u0001\u0000\u0000\u00006\u0178"+
-    "\u0001\u0000\u0000\u00008\u0180\u0001\u0000\u0000\u0000:\u0188\u0001\u0000"+
-    "\u0000\u0000<\u018c\u0001\u0000\u0000\u0000>\u01b8\u0001\u0000\u0000\u0000"+
-    "@\u01bc\u0001\u0000\u0000\u0000B\u01c0\u0001\u0000\u0000\u0000D\u01c2"+
-    "\u0001\u0000\u0000\u0000F\u01c5\u0001\u0000\u0000\u0000H\u01ce\u0001\u0000"+
-    "\u0000\u0000J\u01d6\u0001\u0000\u0000\u0000L\u01d9\u0001\u0000\u0000\u0000"+
-    "N\u01dc\u0001\u0000\u0000\u0000P\u01e5\u0001\u0000\u0000\u0000R\u01e9"+
-    "\u0001\u0000\u0000\u0000T\u01ef\u0001\u0000\u0000\u0000V\u01f3\u0001\u0000"+
-    "\u0000\u0000X\u01f6\u0001\u0000\u0000\u0000Z\u01fe\u0001\u0000\u0000\u0000"+
-    "\\\u0202\u0001\u0000\u0000\u0000^\u0206\u0001\u0000\u0000\u0000`\u0209"+
-    "\u0001\u0000\u0000\u0000b\u020e\u0001\u0000\u0000\u0000d\u0212\u0001\u0000"+
-    "\u0000\u0000f\u0214\u0001\u0000\u0000\u0000h\u0216\u0001\u0000\u0000\u0000"+
-    "j\u0219\u0001\u0000\u0000\u0000l\u021d\u0001\u0000\u0000\u0000n\u0220"+
-    "\u0001\u0000\u0000\u0000p\u0234\u0001\u0000\u0000\u0000r\u0238\u0001\u0000"+
-    "\u0000\u0000t\u023d\u0001\u0000\u0000\u0000vw\u0003\u0002\u0001\u0000"+
-    "wx\u0005\u0000\u0000\u0001x\u0001\u0001\u0000\u0000\u0000yz\u0006\u0001"+
-    "\uffff\uffff\u0000z{\u0003\u0004\u0002\u0000{\u0081\u0001\u0000\u0000"+
-    "\u0000|}\n\u0001\u0000\u0000}~\u0005\u0019\u0000\u0000~\u0080\u0003\u0006"+
-    "\u0003\u0000\u007f|\u0001\u0000\u0000\u0000\u0080\u0083\u0001\u0000\u0000"+
-    "\u0000\u0081\u007f\u0001\u0000\u0000\u0000\u0081\u0082\u0001\u0000\u0000"+
-    "\u0000\u0082\u0003\u0001\u0000\u0000\u0000\u0083\u0081\u0001\u0000\u0000"+
-    "\u0000\u0084\u008b\u0003h4\u0000\u0085\u008b\u0003 \u0010\u0000\u0086"+
-    "\u008b\u0003\u001a\r\u0000\u0087\u008b\u0003l6\u0000\u0088\u0089\u0004"+
-    "\u0002\u0001\u0000\u0089\u008b\u0003.\u0017\u0000\u008a\u0084\u0001\u0000"+
-    "\u0000\u0000\u008a\u0085\u0001\u0000\u0000\u0000\u008a\u0086\u0001\u0000"+
-    "\u0000\u0000\u008a\u0087\u0001\u0000\u0000\u0000\u008a\u0088\u0001\u0000"+
-    "\u0000\u0000\u008b\u0005\u0001\u0000\u0000\u0000\u008c\u009d\u00030\u0018"+
-    "\u0000\u008d\u009d\u0003\b\u0004\u0000\u008e\u009d\u0003J%\u0000\u008f"+
-    "\u009d\u0003D\"\u0000\u0090\u009d\u00032\u0019\u0000\u0091\u009d\u0003"+
-    "F#\u0000\u0092\u009d\u0003L&\u0000\u0093\u009d\u0003N\'\u0000\u0094\u009d"+
-    "\u0003R)\u0000\u0095\u009d\u0003T*\u0000\u0096\u009d\u0003n7\u0000\u0097"+
-    "\u009d\u0003V+\u0000\u0098\u0099\u0004\u0003\u0002\u0000\u0099\u009d\u0003"+
-    "t:\u0000\u009a\u009b\u0004\u0003\u0003\u0000\u009b\u009d\u0003r9\u0000"+
-    "\u009c\u008c\u0001\u0000\u0000\u0000\u009c\u008d\u0001\u0000\u0000\u0000"+
-    "\u009c\u008e\u0001\u0000\u0000\u0000\u009c\u008f\u0001\u0000\u0000\u0000"+
-    "\u009c\u0090\u0001\u0000\u0000\u0000\u009c\u0091\u0001\u0000\u0000\u0000"+
-    "\u009c\u0092\u0001\u0000\u0000\u0000\u009c\u0093\u0001\u0000\u0000\u0000"+
-    "\u009c\u0094\u0001\u0000\u0000\u0000\u009c\u0095\u0001\u0000\u0000\u0000"+
-    "\u009c\u0096\u0001\u0000\u0000\u0000\u009c\u0097\u0001\u0000\u0000\u0000"+
-    "\u009c\u0098\u0001\u0000\u0000\u0000\u009c\u009a\u0001\u0000\u0000\u0000"+
-    "\u009d\u0007\u0001\u0000\u0000\u0000\u009e\u009f\u0005\u0010\u0000\u0000"+
-    "\u009f\u00a0\u0003\n\u0005\u0000\u00a0\t\u0001\u0000\u0000\u0000\u00a1"+
-    "\u00a2\u0006\u0005\uffff\uffff\u0000\u00a2\u00a3\u0005,\u0000\u0000\u00a3"+
-    "\u00c0\u0003\n\u0005\b\u00a4\u00c0\u0003\u0010\b\u0000\u00a5\u00c0\u0003"+
-    "\f\u0006\u0000\u00a6\u00a8\u0003\u0010\b\u0000\u00a7\u00a9\u0005,\u0000"+
-    "\u0000\u00a8\u00a7\u0001\u0000\u0000\u0000\u00a8\u00a9\u0001\u0000\u0000"+
-    "\u0000\u00a9\u00aa\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\'\u0000\u0000"+
-    "\u00ab\u00ac\u0005+\u0000\u0000\u00ac\u00b1\u0003\u0010\b\u0000\u00ad"+
-    "\u00ae\u0005\"\u0000\u0000\u00ae\u00b0\u0003\u0010\b\u0000\u00af\u00ad"+
-    "\u0001\u0000\u0000\u0000\u00b0\u00b3\u0001\u0000\u0000\u0000\u00b1\u00af"+
-    "\u0001\u0000\u0000\u0000\u00b1\u00b2\u0001\u0000\u0000\u0000\u00b2\u00b4"+
-    "\u0001\u0000\u0000\u0000\u00b3\u00b1\u0001\u0000\u0000\u0000\u00b4\u00b5"+
-    "\u00052\u0000\u0000\u00b5\u00c0\u0001\u0000\u0000\u0000\u00b6\u00b7\u0003"+
-    "\u0010\b\u0000\u00b7\u00b9\u0005(\u0000\u0000\u00b8\u00ba\u0005,\u0000"+
-    "\u0000\u00b9\u00b8\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000\u0000"+
-    "\u0000\u00ba\u00bb\u0001\u0000\u0000\u0000\u00bb\u00bc\u0005-\u0000\u0000"+
-    "\u00bc\u00c0\u0001\u0000\u0000\u0000\u00bd\u00be\u0004\u0005\u0004\u0000"+
-    "\u00be\u00c0\u0003\u000e\u0007\u0000\u00bf\u00a1\u0001\u0000\u0000\u0000"+
-    "\u00bf\u00a4\u0001\u0000\u0000\u0000\u00bf\u00a5\u0001\u0000\u0000\u0000"+
-    "\u00bf\u00a6\u0001\u0000\u0000\u0000\u00bf\u00b6\u0001\u0000\u0000\u0000"+
-    "\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c9\u0001\u0000\u0000\u0000"+
-    "\u00c1\u00c2\n\u0005\u0000\u0000\u00c2\u00c3\u0005\u001e\u0000\u0000\u00c3"+
-    "\u00c8\u0003\n\u0005\u0006\u00c4\u00c5\n\u0004\u0000\u0000\u00c5\u00c6"+
-    "\u0005/\u0000\u0000\u00c6\u00c8\u0003\n\u0005\u0005\u00c7\u00c1\u0001"+
-    "\u0000\u0000\u0000\u00c7\u00c4\u0001\u0000\u0000\u0000\u00c8\u00cb\u0001"+
-    "\u0000\u0000\u0000\u00c9\u00c7\u0001\u0000\u0000\u0000\u00c9\u00ca\u0001"+
-    "\u0000\u0000\u0000\u00ca\u000b\u0001\u0000\u0000\u0000\u00cb\u00c9\u0001"+
-    "\u0000\u0000\u0000\u00cc\u00ce\u0003\u0010\b\u0000\u00cd\u00cf\u0005,"+
-    "\u0000\u0000\u00ce\u00cd\u0001\u0000\u0000\u0000\u00ce\u00cf\u0001\u0000"+
-    "\u0000\u0000\u00cf\u00d0\u0001\u0000\u0000\u0000\u00d0\u00d1\u0005*\u0000"+
-    "\u0000\u00d1\u00d2\u0003d2\u0000\u00d2\u00db\u0001\u0000\u0000\u0000\u00d3"+
-    "\u00d5\u0003\u0010\b\u0000\u00d4\u00d6\u0005,\u0000\u0000\u00d5\u00d4"+
-    "\u0001\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7"+
-    "\u0001\u0000\u0000\u0000\u00d7\u00d8\u00051\u0000\u0000\u00d8\u00d9\u0003"+
-    "d2\u0000\u00d9\u00db\u0001\u0000\u0000\u0000\u00da\u00cc\u0001\u0000\u0000"+
-    "\u0000\u00da\u00d3\u0001\u0000\u0000\u0000\u00db\r\u0001\u0000\u0000\u0000"+
-    "\u00dc\u00dd\u0003\u0010\b\u0000\u00dd\u00de\u0005\u0013\u0000\u0000\u00de"+
-    "\u00df\u0003d2\u0000\u00df\u000f\u0001\u0000\u0000\u0000\u00e0\u00e6\u0003"+
-    "\u0012\t\u0000\u00e1\u00e2\u0003\u0012\t\u0000\u00e2\u00e3\u0003f3\u0000"+
-    "\u00e3\u00e4\u0003\u0012\t\u0000\u00e4\u00e6\u0001\u0000\u0000\u0000\u00e5"+
-    "\u00e0\u0001\u0000\u0000\u0000\u00e5\u00e1\u0001\u0000\u0000\u0000\u00e6"+
-    "\u0011\u0001\u0000\u0000\u0000\u00e7\u00e8\u0006\t\uffff\uffff\u0000\u00e8"+
-    "\u00ec\u0003\u0014\n\u0000\u00e9\u00ea\u0007\u0000\u0000\u0000\u00ea\u00ec"+
-    "\u0003\u0012\t\u0003\u00eb\u00e7\u0001\u0000\u0000\u0000\u00eb\u00e9\u0001"+
-    "\u0000\u0000\u0000\u00ec\u00f5\u0001\u0000\u0000\u0000\u00ed\u00ee\n\u0002"+
-    "\u0000\u0000\u00ee\u00ef\u0007\u0001\u0000\u0000\u00ef\u00f4\u0003\u0012"+
-    "\t\u0003\u00f0\u00f1\n\u0001\u0000\u0000\u00f1\u00f2\u0007\u0000\u0000"+
-    "\u0000\u00f2\u00f4\u0003\u0012\t\u0002\u00f3\u00ed\u0001\u0000\u0000\u0000"+
-    "\u00f3\u00f0\u0001\u0000\u0000\u0000\u00f4\u00f7\u0001\u0000\u0000\u0000"+
-    "\u00f5\u00f3\u0001\u0000\u0000\u0000\u00f5\u00f6\u0001\u0000\u0000\u0000"+
-    "\u00f6\u0013\u0001\u0000\u0000\u0000\u00f7\u00f5\u0001\u0000\u0000\u0000"+
-    "\u00f8\u00f9\u0006\n\uffff\uffff\u0000\u00f9\u0101\u0003>\u001f\u0000"+
-    "\u00fa\u0101\u00034\u001a\u0000\u00fb\u0101\u0003\u0016\u000b\u0000\u00fc"+
-    "\u00fd\u0005+\u0000\u0000\u00fd\u00fe\u0003\n\u0005\u0000\u00fe\u00ff"+
-    "\u00052\u0000\u0000\u00ff\u0101\u0001\u0000\u0000\u0000\u0100\u00f8\u0001"+
-    "\u0000\u0000\u0000\u0100\u00fa\u0001\u0000\u0000\u0000\u0100\u00fb\u0001"+
-    "\u0000\u0000\u0000\u0100\u00fc\u0001\u0000\u0000\u0000\u0101\u0107\u0001"+
-    "\u0000\u0000\u0000\u0102\u0103\n\u0001\u0000\u0000\u0103\u0104\u0005!"+
-    "\u0000\u0000\u0104\u0106\u0003\u0018\f\u0000\u0105\u0102\u0001\u0000\u0000"+
-    "\u0000\u0106\u0109\u0001\u0000\u0000\u0000\u0107\u0105\u0001\u0000\u0000"+
-    "\u0000\u0107\u0108\u0001\u0000\u0000\u0000\u0108\u0015\u0001\u0000\u0000"+
-    "\u0000\u0109\u0107\u0001\u0000\u0000\u0000\u010a\u010b\u0003B!\u0000\u010b"+
-    "\u0115\u0005+\u0000\u0000\u010c\u0116\u0005=\u0000\u0000\u010d\u0112\u0003"+
-    "\n\u0005\u0000\u010e\u010f\u0005\"\u0000\u0000\u010f\u0111\u0003\n\u0005"+
-    "\u0000\u0110\u010e\u0001\u0000\u0000\u0000\u0111\u0114\u0001\u0000\u0000"+
-    "\u0000\u0112\u0110\u0001\u0000\u0000\u0000\u0112\u0113\u0001\u0000\u0000"+
-    "\u0000\u0113\u0116\u0001\u0000\u0000\u0000\u0114\u0112\u0001\u0000\u0000"+
-    "\u0000\u0115\u010c\u0001\u0000\u0000\u0000\u0115\u010d\u0001\u0000\u0000"+
-    "\u0000\u0115\u0116\u0001\u0000\u0000\u0000\u0116\u0117\u0001\u0000\u0000"+
-    "\u0000\u0117\u0118\u00052\u0000\u0000\u0118\u0017\u0001\u0000\u0000\u0000"+
-    "\u0119\u011a\u0003:\u001d\u0000\u011a\u0019\u0001\u0000\u0000\u0000\u011b"+
-    "\u011c\u0005\f\u0000\u0000\u011c\u011d\u0003\u001c\u000e\u0000\u011d\u001b"+
-    "\u0001\u0000\u0000\u0000\u011e\u0123\u0003\u001e\u000f\u0000\u011f\u0120"+
-    "\u0005\"\u0000\u0000\u0120\u0122\u0003\u001e\u000f\u0000\u0121\u011f\u0001"+
-    "\u0000\u0000\u0000\u0122\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001"+
-    "\u0000\u0000\u0000\u0123\u0124\u0001\u0000\u0000\u0000\u0124\u001d\u0001"+
-    "\u0000\u0000\u0000\u0125\u0123\u0001\u0000\u0000\u0000\u0126\u012c\u0003"+
-    "\n\u0005\u0000\u0127\u0128\u00034\u001a\u0000\u0128\u0129\u0005 \u0000"+
-    "\u0000\u0129\u012a\u0003\n\u0005\u0000\u012a\u012c\u0001\u0000\u0000\u0000"+
-    "\u012b\u0126\u0001\u0000\u0000\u0000\u012b\u0127\u0001\u0000\u0000\u0000"+
-    "\u012c\u001f\u0001\u0000\u0000\u0000\u012d\u012e\u0005\u0006\u0000\u0000"+
-    "\u012e\u0133\u0003\"\u0011\u0000\u012f\u0130\u0005\"\u0000\u0000\u0130"+
-    "\u0132\u0003\"\u0011\u0000\u0131\u012f\u0001\u0000\u0000\u0000\u0132\u0135"+
-    "\u0001\u0000\u0000\u0000\u0133\u0131\u0001\u0000\u0000\u0000\u0133\u0134"+
-    "\u0001\u0000\u0000\u0000\u0134\u0137\u0001\u0000\u0000\u0000\u0135\u0133"+
-    "\u0001\u0000\u0000\u0000\u0136\u0138\u0003(\u0014\u0000\u0137\u0136\u0001"+
-    "\u0000\u0000\u0000\u0137\u0138\u0001\u0000\u0000\u0000\u0138!\u0001\u0000"+
-    "\u0000\u0000\u0139\u013a\u0003$\u0012\u0000\u013a\u013b\u0005h\u0000\u0000"+
-    "\u013b\u013c\u0003&\u0013\u0000\u013c\u013f\u0001\u0000\u0000\u0000\u013d"+
-    "\u013f\u0003&\u0013\u0000\u013e\u0139\u0001\u0000\u0000\u0000\u013e\u013d"+
-    "\u0001\u0000\u0000\u0000\u013f#\u0001\u0000\u0000\u0000\u0140\u0141\u0005"+
-    "L\u0000\u0000\u0141%\u0001\u0000\u0000\u0000\u0142\u0143\u0007\u0002\u0000"+
-    "\u0000\u0143\'\u0001\u0000\u0000\u0000\u0144\u0147\u0003*\u0015\u0000"+
-    "\u0145\u0147\u0003,\u0016\u0000\u0146\u0144\u0001\u0000\u0000\u0000\u0146"+
-    "\u0145\u0001\u0000\u0000\u0000\u0147)\u0001\u0000\u0000\u0000\u0148\u0149"+
-    "\u0005K\u0000\u0000\u0149\u014e\u0005L\u0000\u0000\u014a\u014b\u0005\""+
-    "\u0000\u0000\u014b\u014d\u0005L\u0000\u0000\u014c\u014a\u0001\u0000\u0000"+
-    "\u0000\u014d\u0150\u0001\u0000\u0000\u0000\u014e\u014c\u0001\u0000\u0000"+
-    "\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f+\u0001\u0000\u0000\u0000"+
-    "\u0150\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0005A\u0000\u0000\u0152"+
-    "\u0153\u0003*\u0015\u0000\u0153\u0154\u0005B\u0000\u0000\u0154-\u0001"+
-    "\u0000\u0000\u0000\u0155\u0156\u0005\u0014\u0000\u0000\u0156\u015b\u0003"+
-    "\"\u0011\u0000\u0157\u0158\u0005\"\u0000\u0000\u0158\u015a\u0003\"\u0011"+
-    "\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u015a\u015d\u0001\u0000\u0000"+
-    "\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000\u0000"+
-    "\u0000\u015c\u015f\u0001\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000"+
-    "\u0000\u015e\u0160\u0003\u001c\u000e\u0000\u015f\u015e\u0001\u0000\u0000"+
-    "\u0000\u015f\u0160\u0001\u0000\u0000\u0000\u0160\u0163\u0001\u0000\u0000"+
-    "\u0000\u0161\u0162\u0005\u001d\u0000\u0000\u0162\u0164\u0003\u001c\u000e"+
-    "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+
-    "\u0000\u0164/\u0001\u0000\u0000\u0000\u0165\u0166\u0005\u0004\u0000\u0000"+
-    "\u0166\u0167\u0003\u001c\u000e\u0000\u01671\u0001\u0000\u0000\u0000\u0168"+
-    "\u016a\u0005\u000f\u0000\u0000\u0169\u016b\u0003\u001c\u000e\u0000\u016a"+
-    "\u0169\u0001\u0000\u0000\u0000\u016a\u016b\u0001\u0000\u0000\u0000\u016b"+
-    "\u016e\u0001\u0000\u0000\u0000\u016c\u016d\u0005\u001d\u0000\u0000\u016d"+
-    "\u016f\u0003\u001c\u000e\u0000\u016e\u016c\u0001\u0000\u0000\u0000\u016e"+
-    "\u016f\u0001\u0000\u0000\u0000\u016f3\u0001\u0000\u0000\u0000\u0170\u0175"+
-    "\u0003B!\u0000\u0171\u0172\u0005$\u0000\u0000\u0172\u0174\u0003B!\u0000"+
-    "\u0173\u0171\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000"+
-    "\u0175\u0173\u0001\u0000\u0000\u0000\u0175\u0176\u0001\u0000\u0000\u0000"+
-    "\u01765\u0001\u0000\u0000\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178"+
-    "\u017d\u0003<\u001e\u0000\u0179\u017a\u0005$\u0000\u0000\u017a\u017c\u0003"+
-    "<\u001e\u0000\u017b\u0179\u0001\u0000\u0000\u0000\u017c\u017f\u0001\u0000"+
-    "\u0000\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017d\u017e\u0001\u0000"+
-    "\u0000\u0000\u017e7\u0001\u0000\u0000\u0000\u017f\u017d\u0001\u0000\u0000"+
-    "\u0000\u0180\u0185\u00036\u001b\u0000\u0181\u0182\u0005\"\u0000\u0000"+
-    "\u0182\u0184\u00036\u001b\u0000\u0183\u0181\u0001\u0000\u0000\u0000\u0184"+
-    "\u0187\u0001\u0000\u0000\u0000\u0185\u0183\u0001\u0000\u0000\u0000\u0185"+
-    "\u0186\u0001\u0000\u0000\u0000\u01869\u0001\u0000\u0000\u0000\u0187\u0185"+
-    "\u0001\u0000\u0000\u0000\u0188\u0189\u0007\u0003\u0000\u0000\u0189;\u0001"+
-    "\u0000\u0000\u0000\u018a\u018d\u0005P\u0000\u0000\u018b\u018d\u0003@ "+
-    "\u0000\u018c\u018a\u0001\u0000\u0000\u0000\u018c\u018b\u0001\u0000\u0000"+
-    "\u0000\u018d=\u0001\u0000\u0000\u0000\u018e\u01b9\u0005-\u0000\u0000\u018f"+
-    "\u0190\u0003b1\u0000\u0190\u0191\u0005C\u0000\u0000\u0191\u01b9\u0001"+
-    "\u0000\u0000\u0000\u0192\u01b9\u0003`0\u0000\u0193\u01b9\u0003b1\u0000"+
-    "\u0194\u01b9\u0003\\.\u0000\u0195\u01b9\u0003@ \u0000\u0196\u01b9\u0003"+
-    "d2\u0000\u0197\u0198\u0005A\u0000\u0000\u0198\u019d\u0003^/\u0000\u0199"+
-    "\u019a\u0005\"\u0000\u0000\u019a\u019c\u0003^/\u0000\u019b\u0199\u0001"+
-    "\u0000\u0000\u0000\u019c\u019f\u0001\u0000\u0000\u0000\u019d\u019b\u0001"+
-    "\u0000\u0000\u0000\u019d\u019e\u0001\u0000\u0000\u0000\u019e\u01a0\u0001"+
-    "\u0000\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u01a0\u01a1\u0005"+
-    "B\u0000\u0000\u01a1\u01b9\u0001\u0000\u0000\u0000\u01a2\u01a3\u0005A\u0000"+
-    "\u0000\u01a3\u01a8\u0003\\.\u0000\u01a4\u01a5\u0005\"\u0000\u0000\u01a5"+
-    "\u01a7\u0003\\.\u0000\u01a6\u01a4\u0001\u0000\u0000\u0000\u01a7\u01aa"+
-    "\u0001\u0000\u0000\u0000\u01a8\u01a6\u0001\u0000\u0000\u0000\u01a8\u01a9"+
-    "\u0001\u0000\u0000\u0000\u01a9\u01ab\u0001\u0000\u0000\u0000\u01aa\u01a8"+
-    "\u0001\u0000\u0000\u0000\u01ab\u01ac\u0005B\u0000\u0000\u01ac\u01b9\u0001"+
-    "\u0000\u0000\u0000\u01ad\u01ae\u0005A\u0000\u0000\u01ae\u01b3\u0003d2"+
-    "\u0000\u01af\u01b0\u0005\"\u0000\u0000\u01b0\u01b2\u0003d2\u0000\u01b1"+
-    "\u01af\u0001\u0000\u0000\u0000\u01b2\u01b5\u0001\u0000\u0000\u0000\u01b3"+
-    "\u01b1\u0001\u0000\u0000\u0000\u01b3\u01b4\u0001\u0000\u0000\u0000\u01b4"+
-    "\u01b6\u0001\u0000\u0000\u0000\u01b5\u01b3\u0001\u0000\u0000\u0000\u01b6"+
-    "\u01b7\u0005B\u0000\u0000\u01b7\u01b9\u0001\u0000\u0000\u0000\u01b8\u018e"+
-    "\u0001\u0000\u0000\u0000\u01b8\u018f\u0001\u0000\u0000\u0000\u01b8\u0192"+
-    "\u0001\u0000\u0000\u0000\u01b8\u0193\u0001\u0000\u0000\u0000\u01b8\u0194"+
-    "\u0001\u0000\u0000\u0000\u01b8\u0195\u0001\u0000\u0000\u0000\u01b8\u0196"+
-    "\u0001\u0000\u0000\u0000\u01b8\u0197\u0001\u0000\u0000\u0000\u01b8\u01a2"+
-    "\u0001\u0000\u0000\u0000\u01b8\u01ad\u0001\u0000\u0000\u0000\u01b9?\u0001"+
-    "\u0000\u0000\u0000\u01ba\u01bd\u00050\u0000\u0000\u01bb\u01bd\u0005@\u0000"+
-    "\u0000\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bc\u01bb\u0001\u0000\u0000"+
-    "\u0000\u01bdA\u0001\u0000\u0000\u0000\u01be\u01c1\u0003:\u001d\u0000\u01bf"+
-    "\u01c1\u0003@ \u0000\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01bf\u0001"+
-    "\u0000\u0000\u0000\u01c1C\u0001\u0000\u0000\u0000\u01c2\u01c3\u0005\t"+
-    "\u0000\u0000\u01c3\u01c4\u0005\u001b\u0000\u0000\u01c4E\u0001\u0000\u0000"+
-    "\u0000\u01c5\u01c6\u0005\u000e\u0000\u0000\u01c6\u01cb\u0003H$\u0000\u01c7"+
-    "\u01c8\u0005\"\u0000\u0000\u01c8\u01ca\u0003H$\u0000\u01c9\u01c7\u0001"+
-    "\u0000\u0000\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001"+
-    "\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01ccG\u0001\u0000"+
-    "\u0000\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01d0\u0003\n\u0005"+
-    "\u0000\u01cf\u01d1\u0007\u0004\u0000\u0000\u01d0\u01cf\u0001\u0000\u0000"+
-    "\u0000\u01d0\u01d1\u0001\u0000\u0000\u0000\u01d1\u01d4\u0001\u0000\u0000"+
-    "\u0000\u01d2\u01d3\u0005.\u0000\u0000\u01d3\u01d5\u0007\u0005\u0000\u0000"+
-    "\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d4\u01d5\u0001\u0000\u0000\u0000"+
-    "\u01d5I\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005\b\u0000\u0000\u01d7"+
-    "\u01d8\u00038\u001c\u0000\u01d8K\u0001\u0000\u0000\u0000\u01d9\u01da\u0005"+
-    "\u0002\u0000\u0000\u01da\u01db\u00038\u001c\u0000\u01dbM\u0001\u0000\u0000"+
-    "\u0000\u01dc\u01dd\u0005\u000b\u0000\u0000\u01dd\u01e2\u0003P(\u0000\u01de"+
-    "\u01df\u0005\"\u0000\u0000\u01df\u01e1\u0003P(\u0000\u01e0\u01de\u0001"+
-    "\u0000\u0000\u0000\u01e1\u01e4\u0001\u0000\u0000\u0000\u01e2\u01e0\u0001"+
-    "\u0000\u0000\u0000\u01e2\u01e3\u0001\u0000\u0000\u0000\u01e3O\u0001\u0000"+
-    "\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5\u01e6\u00036\u001b"+
-    "\u0000\u01e6\u01e7\u0005T\u0000\u0000\u01e7\u01e8\u00036\u001b\u0000\u01e8"+
-    "Q\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005\u0001\u0000\u0000\u01ea\u01eb"+
-    "\u0003\u0014\n\u0000\u01eb\u01ed\u0003d2\u0000\u01ec\u01ee\u0003X,\u0000"+
-    "\u01ed\u01ec\u0001\u0000\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000\u0000"+
-    "\u01eeS\u0001\u0000\u0000\u0000\u01ef\u01f0\u0005\u0007\u0000\u0000\u01f0"+
-    "\u01f1\u0003\u0014\n\u0000\u01f1\u01f2\u0003d2\u0000\u01f2U\u0001\u0000"+
-    "\u0000\u0000\u01f3\u01f4\u0005\n\u0000\u0000\u01f4\u01f5\u00034\u001a"+
-    "\u0000\u01f5W\u0001\u0000\u0000\u0000\u01f6\u01fb\u0003Z-\u0000\u01f7"+
-    "\u01f8\u0005\"\u0000\u0000\u01f8\u01fa\u0003Z-\u0000\u01f9\u01f7\u0001"+
-    "\u0000\u0000\u0000\u01fa\u01fd\u0001\u0000\u0000\u0000\u01fb\u01f9\u0001"+
-    "\u0000\u0000\u0000\u01fb\u01fc\u0001\u0000\u0000\u0000\u01fcY\u0001\u0000"+
-    "\u0000\u0000\u01fd\u01fb\u0001\u0000\u0000\u0000\u01fe\u01ff\u0003:\u001d"+
-    "\u0000\u01ff\u0200\u0005 \u0000\u0000\u0200\u0201\u0003>\u001f\u0000\u0201"+
-    "[\u0001\u0000\u0000\u0000\u0202\u0203\u0007\u0006\u0000\u0000\u0203]\u0001"+
-    "\u0000\u0000\u0000\u0204\u0207\u0003`0\u0000\u0205\u0207\u0003b1\u0000"+
-    "\u0206\u0204\u0001\u0000\u0000\u0000\u0206\u0205\u0001\u0000\u0000\u0000"+
-    "\u0207_\u0001\u0000\u0000\u0000\u0208\u020a\u0007\u0000\u0000\u0000\u0209"+
-    "\u0208\u0001\u0000\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000\u020a"+
-    "\u020b\u0001\u0000\u0000\u0000\u020b\u020c\u0005\u001c\u0000\u0000\u020c"+
-    "a\u0001\u0000\u0000\u0000\u020d\u020f\u0007\u0000\u0000\u0000\u020e\u020d"+
-    "\u0001\u0000\u0000\u0000\u020e\u020f\u0001\u0000\u0000\u0000\u020f\u0210"+
-    "\u0001\u0000\u0000\u0000\u0210\u0211\u0005\u001b\u0000\u0000\u0211c\u0001"+
-    "\u0000\u0000\u0000\u0212\u0213\u0005\u001a\u0000\u0000\u0213e\u0001\u0000"+
-    "\u0000\u0000\u0214\u0215\u0007\u0007\u0000\u0000\u0215g\u0001\u0000\u0000"+
-    "\u0000\u0216\u0217\u0005\u0005\u0000\u0000\u0217\u0218\u0003j5\u0000\u0218"+
-    "i\u0001\u0000\u0000\u0000\u0219\u021a\u0005A\u0000\u0000\u021a\u021b\u0003"+
-    "\u0002\u0001\u0000\u021b\u021c\u0005B\u0000\u0000\u021ck\u0001\u0000\u0000"+
-    "\u0000\u021d\u021e\u0005\r\u0000\u0000\u021e\u021f\u0005d\u0000\u0000"+
-    "\u021fm\u0001\u0000\u0000\u0000\u0220\u0221\u0005\u0003\u0000\u0000\u0221"+
-    "\u0224\u0005Z\u0000\u0000\u0222\u0223\u0005X\u0000\u0000\u0223\u0225\u0003"+
-    "6\u001b\u0000\u0224\u0222\u0001\u0000\u0000\u0000\u0224\u0225\u0001\u0000"+
-    "\u0000\u0000\u0225\u022f\u0001\u0000\u0000\u0000\u0226\u0227\u0005Y\u0000"+
-    "\u0000\u0227\u022c\u0003p8\u0000\u0228\u0229\u0005\"\u0000\u0000\u0229"+
-    "\u022b\u0003p8\u0000\u022a\u0228\u0001\u0000\u0000\u0000\u022b\u022e\u0001"+
-    "\u0000\u0000\u0000\u022c\u022a\u0001\u0000\u0000\u0000\u022c\u022d\u0001"+
-    "\u0000\u0000\u0000\u022d\u0230\u0001\u0000\u0000\u0000\u022e\u022c\u0001"+
-    "\u0000\u0000\u0000\u022f\u0226\u0001\u0000\u0000\u0000\u022f\u0230\u0001"+
-    "\u0000\u0000\u0000\u0230o\u0001\u0000\u0000\u0000\u0231\u0232\u00036\u001b"+
-    "\u0000\u0232\u0233\u0005 \u0000\u0000\u0233\u0235\u0001\u0000\u0000\u0000"+
-    "\u0234\u0231\u0001\u0000\u0000\u0000\u0234\u0235\u0001\u0000\u0000\u0000"+
-    "\u0235\u0236\u0001\u0000\u0000\u0000\u0236\u0237\u00036\u001b\u0000\u0237"+
-    "q\u0001\u0000\u0000\u0000\u0238\u0239\u0005\u0012\u0000\u0000\u0239\u023a"+
-    "\u0003\"\u0011\u0000\u023a\u023b\u0005X\u0000\u0000\u023b\u023c\u0003"+
-    "8\u001c\u0000\u023cs\u0001\u0000\u0000\u0000\u023d\u023e\u0005\u0011\u0000"+
-    "\u0000\u023e\u0241\u0003\u001c\u000e\u0000\u023f\u0240\u0005\u001d\u0000"+
-    "\u0000\u0240\u0242\u0003\u001c\u000e\u0000\u0241\u023f\u0001\u0000\u0000"+
-    "\u0000\u0241\u0242\u0001\u0000\u0000\u0000\u0242u\u0001\u0000\u0000\u0000"+
-    "8\u0081\u008a\u009c\u00a8\u00b1\u00b9\u00bf\u00c7\u00c9\u00ce\u00d5\u00da"+
-    "\u00e5\u00eb\u00f3\u00f5\u0100\u0107\u0112\u0115\u0123\u012b\u0133\u0137"+
-    "\u013e\u0146\u014e\u015b\u015f\u0163\u016a\u016e\u0175\u017d\u0185\u018c"+
-    "\u019d\u01a8\u01b3\u01b8\u01bc\u01c0\u01cb\u01d0\u01d4\u01e2\u01ed\u01fb"+
-    "\u0206\u0209\u020e\u0224\u022c\u022f\u0234\u0241";
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u00a3\b\u0003\u0001"+
+    "\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
+    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00af\b\u0005\u0001"+
+    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00b6"+
+    "\b\u0005\n\u0005\f\u0005\u00b9\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0003\u0005\u00c0\b\u0005\u0001\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0003\u0005\u00c6\b\u0005\u0001\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00ce\b\u0005"+
+    "\n\u0005\f\u0005\u00d1\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00d5"+
+    "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003"+
+    "\u0006\u00dc\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e1"+
+    "\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001"+
+    "\b\u0001\b\u0001\b\u0001\b\u0003\b\u00ec\b\b\u0001\t\u0001\t\u0001\t\u0001"+
+    "\t\u0003\t\u00f2\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005"+
+    "\t\u00fa\b\t\n\t\f\t\u00fd\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n"+
+    "\u0001\n\u0001\n\u0001\n\u0003\n\u0107\b\n\u0001\n\u0001\n\u0001\n\u0005"+
+    "\n\u010c\b\n\n\n\f\n\u010f\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+
+    "\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0117\b\u000b\n\u000b\f\u000b"+
+    "\u011a\t\u000b\u0003\u000b\u011c\b\u000b\u0001\u000b\u0001\u000b\u0001"+
+    "\f\u0001\f\u0003\f\u0122\b\f\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001"+
+    "\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0005\u000f\u012c\b\u000f\n"+
+    "\u000f\f\u000f\u012f\t\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0003"+
+    "\u0010\u0134\b\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001"+
+    "\u0011\u0001\u0011\u0005\u0011\u013c\b\u0011\n\u0011\f\u0011\u013f\t\u0011"+
+    "\u0001\u0011\u0003\u0011\u0142\b\u0011\u0001\u0012\u0001\u0012\u0001\u0012"+
+    "\u0003\u0012\u0147\b\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+
+    "\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015\u0003\u0015\u0151\b\u0015"+
+    "\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0157\b\u0016"+
+    "\n\u0016\f\u0016\u015a\t\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+
+    "\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018\u0164"+
+    "\b\u0018\n\u0018\f\u0018\u0167\t\u0018\u0001\u0018\u0003\u0018\u016a\b"+
+    "\u0018\u0001\u0018\u0001\u0018\u0003\u0018\u016e\b\u0018\u0001\u0019\u0001"+
+    "\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0003\u001a\u0175\b\u001a\u0001"+
+    "\u001a\u0001\u001a\u0003\u001a\u0179\b\u001a\u0001\u001b\u0001\u001b\u0001"+
+    "\u001b\u0005\u001b\u017e\b\u001b\n\u001b\f\u001b\u0181\t\u001b\u0001\u001c"+
+    "\u0001\u001c\u0001\u001c\u0003\u001c\u0186\b\u001c\u0001\u001d\u0001\u001d"+
+    "\u0001\u001d\u0005\u001d\u018b\b\u001d\n\u001d\f\u001d\u018e\t\u001d\u0001"+
+    "\u001e\u0001\u001e\u0001\u001e\u0005\u001e\u0193\b\u001e\n\u001e\f\u001e"+
+    "\u0196\t\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u019b\b"+
+    "\u001f\n\u001f\f\u001f\u019e\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001"+
+    "!\u0003!\u01a5\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+
+    "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01b4\b\"\n"+
+    "\"\f\"\u01b7\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005"+
+    "\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\""+
+    "\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001\"\u0001\"\u0003\"\u01d1"+
+    "\b\"\u0001#\u0001#\u0003#\u01d5\b#\u0001$\u0001$\u0001$\u0003$\u01da\b"+
+    "$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0005&\u01e3\b&\n&"+
+    "\f&\u01e6\t&\u0001\'\u0001\'\u0003\'\u01ea\b\'\u0001\'\u0001\'\u0003\'"+
+    "\u01ee\b\'\u0001(\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001"+
+    "*\u0001*\u0005*\u01fa\b*\n*\f*\u01fd\t*\u0001+\u0001+\u0001+\u0001+\u0001"+
+    ",\u0001,\u0001,\u0001,\u0003,\u0207\b,\u0001-\u0001-\u0001-\u0001-\u0001"+
+    ".\u0001.\u0001.\u0001/\u0001/\u0001/\u0005/\u0213\b/\n/\f/\u0216\t/\u0001"+
+    "0\u00010\u00010\u00010\u00011\u00011\u00012\u00012\u00032\u0220\b2\u0001"+
+    "3\u00033\u0223\b3\u00013\u00013\u00014\u00034\u0228\b4\u00014\u00014\u0001"+
+    "5\u00015\u00016\u00016\u00017\u00017\u00017\u00018\u00018\u00018\u0001"+
+    "8\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u023e\b:\u0001"+
+    ":\u0001:\u0001:\u0001:\u0005:\u0244\b:\n:\f:\u0247\t:\u0003:\u0249\b:"+
+    "\u0001;\u0001;\u0001;\u0003;\u024e\b;\u0001;\u0001;\u0001<\u0001<\u0001"+
+    "<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0003=\u025b\b=\u0001=\u0000"+
+    "\u0004\u0002\n\u0012\u0014>\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+
+    "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+
+    "TVXZ\\^`bdfhjlnprtvxz\u0000\b\u0001\u0000:;\u0001\u0000<>\u0002\u0000"+
+    "\u0019\u0019LL\u0001\u0000CD\u0002\u0000\u001e\u001e\"\"\u0002\u0000%"+
+    "%((\u0002\u0000$$22\u0002\u00003359\u0277\u0000|\u0001\u0000\u0000\u0000"+
+    "\u0002\u007f\u0001\u0000\u0000\u0000\u0004\u0090\u0001\u0000\u0000\u0000"+
+    "\u0006\u00a2\u0001\u0000\u0000\u0000\b\u00a4\u0001\u0000\u0000\u0000\n"+
+    "\u00c5\u0001\u0000\u0000\u0000\f\u00e0\u0001\u0000\u0000\u0000\u000e\u00e2"+
+    "\u0001\u0000\u0000\u0000\u0010\u00eb\u0001\u0000\u0000\u0000\u0012\u00f1"+
+    "\u0001\u0000\u0000\u0000\u0014\u0106\u0001\u0000\u0000\u0000\u0016\u0110"+
+    "\u0001\u0000\u0000\u0000\u0018\u0121\u0001\u0000\u0000\u0000\u001a\u0123"+
+    "\u0001\u0000\u0000\u0000\u001c\u0125\u0001\u0000\u0000\u0000\u001e\u0128"+
+    "\u0001\u0000\u0000\u0000 \u0133\u0001\u0000\u0000\u0000\"\u0137\u0001"+
+    "\u0000\u0000\u0000$\u0146\u0001\u0000\u0000\u0000&\u014a\u0001\u0000\u0000"+
+    "\u0000(\u014c\u0001\u0000\u0000\u0000*\u0150\u0001\u0000\u0000\u0000,"+
+    "\u0152\u0001\u0000\u0000\u0000.\u015b\u0001\u0000\u0000\u00000\u015f\u0001"+
+    "\u0000\u0000\u00002\u016f\u0001\u0000\u0000\u00004\u0172\u0001\u0000\u0000"+
+    "\u00006\u017a\u0001\u0000\u0000\u00008\u0182\u0001\u0000\u0000\u0000:"+
+    "\u0187\u0001\u0000\u0000\u0000<\u018f\u0001\u0000\u0000\u0000>\u0197\u0001"+
+    "\u0000\u0000\u0000@\u019f\u0001\u0000\u0000\u0000B\u01a4\u0001\u0000\u0000"+
+    "\u0000D\u01d0\u0001\u0000\u0000\u0000F\u01d4\u0001\u0000\u0000\u0000H"+
+    "\u01d9\u0001\u0000\u0000\u0000J\u01db\u0001\u0000\u0000\u0000L\u01de\u0001"+
+    "\u0000\u0000\u0000N\u01e7\u0001\u0000\u0000\u0000P\u01ef\u0001\u0000\u0000"+
+    "\u0000R\u01f2\u0001\u0000\u0000\u0000T\u01f5\u0001\u0000\u0000\u0000V"+
+    "\u01fe\u0001\u0000\u0000\u0000X\u0202\u0001\u0000\u0000\u0000Z\u0208\u0001"+
+    "\u0000\u0000\u0000\\\u020c\u0001\u0000\u0000\u0000^\u020f\u0001\u0000"+
+    "\u0000\u0000`\u0217\u0001\u0000\u0000\u0000b\u021b\u0001\u0000\u0000\u0000"+
+    "d\u021f\u0001\u0000\u0000\u0000f\u0222\u0001\u0000\u0000\u0000h\u0227"+
+    "\u0001\u0000\u0000\u0000j\u022b\u0001\u0000\u0000\u0000l\u022d\u0001\u0000"+
+    "\u0000\u0000n\u022f\u0001\u0000\u0000\u0000p\u0232\u0001\u0000\u0000\u0000"+
+    "r\u0236\u0001\u0000\u0000\u0000t\u0239\u0001\u0000\u0000\u0000v\u024d"+
+    "\u0001\u0000\u0000\u0000x\u0251\u0001\u0000\u0000\u0000z\u0256\u0001\u0000"+
+    "\u0000\u0000|}\u0003\u0002\u0001\u0000}~\u0005\u0000\u0000\u0001~\u0001"+
+    "\u0001\u0000\u0000\u0000\u007f\u0080\u0006\u0001\uffff\uffff\u0000\u0080"+
+    "\u0081\u0003\u0004\u0002\u0000\u0081\u0087\u0001\u0000\u0000\u0000\u0082"+
+    "\u0083\n\u0001\u0000\u0000\u0083\u0084\u0005\u0018\u0000\u0000\u0084\u0086"+
+    "\u0003\u0006\u0003\u0000\u0085\u0082\u0001\u0000\u0000\u0000\u0086\u0089"+
+    "\u0001\u0000\u0000\u0000\u0087\u0085\u0001\u0000\u0000\u0000\u0087\u0088"+
+    "\u0001\u0000\u0000\u0000\u0088\u0003\u0001\u0000\u0000\u0000\u0089\u0087"+
+    "\u0001\u0000\u0000\u0000\u008a\u0091\u0003n7\u0000\u008b\u0091\u0003\""+
+    "\u0011\u0000\u008c\u0091\u0003\u001c\u000e\u0000\u008d\u0091\u0003r9\u0000"+
+    "\u008e\u008f\u0004\u0002\u0001\u0000\u008f\u0091\u00030\u0018\u0000\u0090"+
+    "\u008a\u0001\u0000\u0000\u0000\u0090\u008b\u0001\u0000\u0000\u0000\u0090"+
+    "\u008c\u0001\u0000\u0000\u0000\u0090\u008d\u0001\u0000\u0000\u0000\u0090"+
+    "\u008e\u0001\u0000\u0000\u0000\u0091\u0005\u0001\u0000\u0000\u0000\u0092"+
+    "\u00a3\u00032\u0019\u0000\u0093\u00a3\u0003\b\u0004\u0000\u0094\u00a3"+
+    "\u0003P(\u0000\u0095\u00a3\u0003J%\u0000\u0096\u00a3\u00034\u001a\u0000"+
+    "\u0097\u00a3\u0003L&\u0000\u0098\u00a3\u0003R)\u0000\u0099\u00a3\u0003"+
+    "T*\u0000\u009a\u00a3\u0003X,\u0000\u009b\u00a3\u0003Z-\u0000\u009c\u00a3"+
+    "\u0003t:\u0000\u009d\u00a3\u0003\\.\u0000\u009e\u009f\u0004\u0003\u0002"+
+    "\u0000\u009f\u00a3\u0003z=\u0000\u00a0\u00a1\u0004\u0003\u0003\u0000\u00a1"+
+    "\u00a3\u0003x<\u0000\u00a2\u0092\u0001\u0000\u0000\u0000\u00a2\u0093\u0001"+
+    "\u0000\u0000\u0000\u00a2\u0094\u0001\u0000\u0000\u0000\u00a2\u0095\u0001"+
+    "\u0000\u0000\u0000\u00a2\u0096\u0001\u0000\u0000\u0000\u00a2\u0097\u0001"+
+    "\u0000\u0000\u0000\u00a2\u0098\u0001\u0000\u0000\u0000\u00a2\u0099\u0001"+
+    "\u0000\u0000\u0000\u00a2\u009a\u0001\u0000\u0000\u0000\u00a2\u009b\u0001"+
+    "\u0000\u0000\u0000\u00a2\u009c\u0001\u0000\u0000\u0000\u00a2\u009d\u0001"+
+    "\u0000\u0000\u0000\u00a2\u009e\u0001\u0000\u0000\u0000\u00a2\u00a0\u0001"+
+    "\u0000\u0000\u0000\u00a3\u0007\u0001\u0000\u0000\u0000\u00a4\u00a5\u0005"+
+    "\u0010\u0000\u0000\u00a5\u00a6\u0003\n\u0005\u0000\u00a6\t\u0001\u0000"+
+    "\u0000\u0000\u00a7\u00a8\u0006\u0005\uffff\uffff\u0000\u00a8\u00a9\u0005"+
+    "+\u0000\u0000\u00a9\u00c6\u0003\n\u0005\b\u00aa\u00c6\u0003\u0010\b\u0000"+
+    "\u00ab\u00c6\u0003\f\u0006\u0000\u00ac\u00ae\u0003\u0010\b\u0000\u00ad"+
+    "\u00af\u0005+\u0000\u0000\u00ae\u00ad\u0001\u0000\u0000\u0000\u00ae\u00af"+
+    "\u0001\u0000\u0000\u0000\u00af\u00b0\u0001\u0000\u0000\u0000\u00b0\u00b1"+
+    "\u0005&\u0000\u0000\u00b1\u00b2\u0005*\u0000\u0000\u00b2\u00b7\u0003\u0010"+
+    "\b\u0000\u00b3\u00b4\u0005!\u0000\u0000\u00b4\u00b6\u0003\u0010\b\u0000"+
+    "\u00b5\u00b3\u0001\u0000\u0000\u0000\u00b6\u00b9\u0001\u0000\u0000\u0000"+
+    "\u00b7\u00b5\u0001\u0000\u0000\u0000\u00b7\u00b8\u0001\u0000\u0000\u0000"+
+    "\u00b8\u00ba\u0001\u0000\u0000\u0000\u00b9\u00b7\u0001\u0000\u0000\u0000"+
+    "\u00ba\u00bb\u00051\u0000\u0000\u00bb\u00c6\u0001\u0000\u0000\u0000\u00bc"+
+    "\u00bd\u0003\u0010\b\u0000\u00bd\u00bf\u0005\'\u0000\u0000\u00be\u00c0"+
+    "\u0005+\u0000\u0000\u00bf\u00be\u0001\u0000\u0000\u0000\u00bf\u00c0\u0001"+
+    "\u0000\u0000\u0000\u00c0\u00c1\u0001\u0000\u0000\u0000\u00c1\u00c2\u0005"+
+    ",\u0000\u0000\u00c2\u00c6\u0001\u0000\u0000\u0000\u00c3\u00c4\u0004\u0005"+
+    "\u0004\u0000\u00c4\u00c6\u0003\u000e\u0007\u0000\u00c5\u00a7\u0001\u0000"+
+    "\u0000\u0000\u00c5\u00aa\u0001\u0000\u0000\u0000\u00c5\u00ab\u0001\u0000"+
+    "\u0000\u0000\u00c5\u00ac\u0001\u0000\u0000\u0000\u00c5\u00bc\u0001\u0000"+
+    "\u0000\u0000\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c6\u00cf\u0001\u0000"+
+    "\u0000\u0000\u00c7\u00c8\n\u0005\u0000\u0000\u00c8\u00c9\u0005\u001d\u0000"+
+    "\u0000\u00c9\u00ce\u0003\n\u0005\u0006\u00ca\u00cb\n\u0004\u0000\u0000"+
+    "\u00cb\u00cc\u0005.\u0000\u0000\u00cc\u00ce\u0003\n\u0005\u0005\u00cd"+
+    "\u00c7\u0001\u0000\u0000\u0000\u00cd\u00ca\u0001\u0000\u0000\u0000\u00ce"+
+    "\u00d1\u0001\u0000\u0000\u0000\u00cf\u00cd\u0001\u0000\u0000\u0000\u00cf"+
+    "\u00d0\u0001\u0000\u0000\u0000\u00d0\u000b\u0001\u0000\u0000\u0000\u00d1"+
+    "\u00cf\u0001\u0000\u0000\u0000\u00d2\u00d4\u0003\u0010\b\u0000\u00d3\u00d5"+
+    "\u0005+\u0000\u0000\u00d4\u00d3\u0001\u0000\u0000\u0000\u00d4\u00d5\u0001"+
+    "\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7\u0005"+
+    ")\u0000\u0000\u00d7\u00d8\u0003j5\u0000\u00d8\u00e1\u0001\u0000\u0000"+
+    "\u0000\u00d9\u00db\u0003\u0010\b\u0000\u00da\u00dc\u0005+\u0000\u0000"+
+    "\u00db\u00da\u0001\u0000\u0000\u0000\u00db\u00dc\u0001\u0000\u0000\u0000"+
+    "\u00dc\u00dd\u0001\u0000\u0000\u0000\u00dd\u00de\u00050\u0000\u0000\u00de"+
+    "\u00df\u0003j5\u0000\u00df\u00e1\u0001\u0000\u0000\u0000\u00e0\u00d2\u0001"+
+    "\u0000\u0000\u0000\u00e0\u00d9\u0001\u0000\u0000\u0000\u00e1\r\u0001\u0000"+
+    "\u0000\u0000\u00e2\u00e3\u0003\u0010\b\u0000\u00e3\u00e4\u0005?\u0000"+
+    "\u0000\u00e4\u00e5\u0003j5\u0000\u00e5\u000f\u0001\u0000\u0000\u0000\u00e6"+
+    "\u00ec\u0003\u0012\t\u0000\u00e7\u00e8\u0003\u0012\t\u0000\u00e8\u00e9"+
+    "\u0003l6\u0000\u00e9\u00ea\u0003\u0012\t\u0000\u00ea\u00ec\u0001\u0000"+
+    "\u0000\u0000\u00eb\u00e6\u0001\u0000\u0000\u0000\u00eb\u00e7\u0001\u0000"+
+    "\u0000\u0000\u00ec\u0011\u0001\u0000\u0000\u0000\u00ed\u00ee\u0006\t\uffff"+
+    "\uffff\u0000\u00ee\u00f2\u0003\u0014\n\u0000\u00ef\u00f0\u0007\u0000\u0000"+
+    "\u0000\u00f0\u00f2\u0003\u0012\t\u0003\u00f1\u00ed\u0001\u0000\u0000\u0000"+
+    "\u00f1\u00ef\u0001\u0000\u0000\u0000\u00f2\u00fb\u0001\u0000\u0000\u0000"+
+    "\u00f3\u00f4\n\u0002\u0000\u0000\u00f4\u00f5\u0007\u0001\u0000\u0000\u00f5"+
+    "\u00fa\u0003\u0012\t\u0003\u00f6\u00f7\n\u0001\u0000\u0000\u00f7\u00f8"+
+    "\u0007\u0000\u0000\u0000\u00f8\u00fa\u0003\u0012\t\u0002\u00f9\u00f3\u0001"+
+    "\u0000\u0000\u0000\u00f9\u00f6\u0001\u0000\u0000\u0000\u00fa\u00fd\u0001"+
+    "\u0000\u0000\u0000\u00fb\u00f9\u0001\u0000\u0000\u0000\u00fb\u00fc\u0001"+
+    "\u0000\u0000\u0000\u00fc\u0013\u0001\u0000\u0000\u0000\u00fd\u00fb\u0001"+
+    "\u0000\u0000\u0000\u00fe\u00ff\u0006\n\uffff\uffff\u0000\u00ff\u0107\u0003"+
+    "D\"\u0000\u0100\u0107\u0003:\u001d\u0000\u0101\u0107\u0003\u0016\u000b"+
+    "\u0000\u0102\u0103\u0005*\u0000\u0000\u0103\u0104\u0003\n\u0005\u0000"+
+    "\u0104\u0105\u00051\u0000\u0000\u0105\u0107\u0001\u0000\u0000\u0000\u0106"+
+    "\u00fe\u0001\u0000\u0000\u0000\u0106\u0100\u0001\u0000\u0000\u0000\u0106"+
+    "\u0101\u0001\u0000\u0000\u0000\u0106\u0102\u0001\u0000\u0000\u0000\u0107"+
+    "\u010d\u0001\u0000\u0000\u0000\u0108\u0109\n\u0001\u0000\u0000\u0109\u010a"+
+    "\u0005 \u0000\u0000\u010a\u010c\u0003\u001a\r\u0000\u010b\u0108\u0001"+
+    "\u0000\u0000\u0000\u010c\u010f\u0001\u0000\u0000\u0000\u010d\u010b\u0001"+
+    "\u0000\u0000\u0000\u010d\u010e\u0001\u0000\u0000\u0000\u010e\u0015\u0001"+
+    "\u0000\u0000\u0000\u010f\u010d\u0001\u0000\u0000\u0000\u0110\u0111\u0003"+
+    "\u0018\f\u0000\u0111\u011b\u0005*\u0000\u0000\u0112\u011c\u0005<\u0000"+
+    "\u0000\u0113\u0118\u0003\n\u0005\u0000\u0114\u0115\u0005!\u0000\u0000"+
+    "\u0115\u0117\u0003\n\u0005\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0117"+
+    "\u011a\u0001\u0000\u0000\u0000\u0118\u0116\u0001\u0000\u0000\u0000\u0118"+
+    "\u0119\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000\u011a"+
+    "\u0118\u0001\u0000\u0000\u0000\u011b\u0112\u0001\u0000\u0000\u0000\u011b"+
+    "\u0113\u0001\u0000\u0000\u0000\u011b\u011c\u0001\u0000\u0000\u0000\u011c"+
+    "\u011d\u0001\u0000\u0000\u0000\u011d\u011e\u00051\u0000\u0000\u011e\u0017"+
+    "\u0001\u0000\u0000\u0000\u011f\u0122\u0005?\u0000\u0000\u0120\u0122\u0003"+
+    "H$\u0000\u0121\u011f\u0001\u0000\u0000\u0000\u0121\u0120\u0001\u0000\u0000"+
+    "\u0000\u0122\u0019\u0001\u0000\u0000\u0000\u0123\u0124\u0003@ \u0000\u0124"+
+    "\u001b\u0001\u0000\u0000\u0000\u0125\u0126\u0005\f\u0000\u0000\u0126\u0127"+
+    "\u0003\u001e\u000f\u0000\u0127\u001d\u0001\u0000\u0000\u0000\u0128\u012d"+
+    "\u0003 \u0010\u0000\u0129\u012a\u0005!\u0000\u0000\u012a\u012c\u0003 "+
+    "\u0010\u0000\u012b\u0129\u0001\u0000\u0000\u0000\u012c\u012f\u0001\u0000"+
+    "\u0000\u0000\u012d\u012b\u0001\u0000\u0000\u0000\u012d\u012e\u0001\u0000"+
+    "\u0000\u0000\u012e\u001f\u0001\u0000\u0000\u0000\u012f\u012d\u0001\u0000"+
+    "\u0000\u0000\u0130\u0131\u0003:\u001d\u0000\u0131\u0132\u0005\u001f\u0000"+
+    "\u0000\u0132\u0134\u0001\u0000\u0000\u0000\u0133\u0130\u0001\u0000\u0000"+
+    "\u0000\u0133\u0134\u0001\u0000\u0000\u0000\u0134\u0135\u0001\u0000\u0000"+
+    "\u0000\u0135\u0136\u0003\n\u0005\u0000\u0136!\u0001\u0000\u0000\u0000"+
+    "\u0137\u0138\u0005\u0006\u0000\u0000\u0138\u013d\u0003$\u0012\u0000\u0139"+
+    "\u013a\u0005!\u0000\u0000\u013a\u013c\u0003$\u0012\u0000\u013b\u0139\u0001"+
+    "\u0000\u0000\u0000\u013c\u013f\u0001\u0000\u0000\u0000\u013d\u013b\u0001"+
+    "\u0000\u0000\u0000\u013d\u013e\u0001\u0000\u0000\u0000\u013e\u0141\u0001"+
+    "\u0000\u0000\u0000\u013f\u013d\u0001\u0000\u0000\u0000\u0140\u0142\u0003"+
+    "*\u0015\u0000\u0141\u0140\u0001\u0000\u0000\u0000\u0141\u0142\u0001\u0000"+
+    "\u0000\u0000\u0142#\u0001\u0000\u0000\u0000\u0143\u0144\u0003&\u0013\u0000"+
+    "\u0144\u0145\u0005h\u0000\u0000\u0145\u0147\u0001\u0000\u0000\u0000\u0146"+
+    "\u0143\u0001\u0000\u0000\u0000\u0146\u0147\u0001\u0000\u0000\u0000\u0147"+
+    "\u0148\u0001\u0000\u0000\u0000\u0148\u0149\u0003(\u0014\u0000\u0149%\u0001"+
+    "\u0000\u0000\u0000\u014a\u014b\u0005L\u0000\u0000\u014b\'\u0001\u0000"+
+    "\u0000\u0000\u014c\u014d\u0007\u0002\u0000\u0000\u014d)\u0001\u0000\u0000"+
+    "\u0000\u014e\u0151\u0003,\u0016\u0000\u014f\u0151\u0003.\u0017\u0000\u0150"+
+    "\u014e\u0001\u0000\u0000\u0000\u0150\u014f\u0001\u0000\u0000\u0000\u0151"+
+    "+\u0001\u0000\u0000\u0000\u0152\u0153\u0005K\u0000\u0000\u0153\u0158\u0005"+
+    "L\u0000\u0000\u0154\u0155\u0005!\u0000\u0000\u0155\u0157\u0005L\u0000"+
+    "\u0000\u0156\u0154\u0001\u0000\u0000\u0000\u0157\u015a\u0001\u0000\u0000"+
+    "\u0000\u0158\u0156\u0001\u0000\u0000\u0000\u0158\u0159\u0001\u0000\u0000"+
+    "\u0000\u0159-\u0001\u0000\u0000\u0000\u015a\u0158\u0001\u0000\u0000\u0000"+
+    "\u015b\u015c\u0005A\u0000\u0000\u015c\u015d\u0003,\u0016\u0000\u015d\u015e"+
+    "\u0005B\u0000\u0000\u015e/\u0001\u0000\u0000\u0000\u015f\u0160\u0005\u0013"+
+    "\u0000\u0000\u0160\u0165\u0003$\u0012\u0000\u0161\u0162\u0005!\u0000\u0000"+
+    "\u0162\u0164\u0003$\u0012\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0164"+
+    "\u0167\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000\u0165"+
+    "\u0166\u0001\u0000\u0000\u0000\u0166\u0169\u0001\u0000\u0000\u0000\u0167"+
+    "\u0165\u0001\u0000\u0000\u0000\u0168\u016a\u00036\u001b\u0000\u0169\u0168"+
+    "\u0001\u0000\u0000\u0000\u0169\u016a\u0001\u0000\u0000\u0000\u016a\u016d"+
+    "\u0001\u0000\u0000\u0000\u016b\u016c\u0005\u001c\u0000\u0000\u016c\u016e"+
+    "\u0003\u001e\u000f\u0000\u016d\u016b\u0001\u0000\u0000\u0000\u016d\u016e"+
+    "\u0001\u0000\u0000\u0000\u016e1\u0001\u0000\u0000\u0000\u016f\u0170\u0005"+
+    "\u0004\u0000\u0000\u0170\u0171\u0003\u001e\u000f\u0000\u01713\u0001\u0000"+
+    "\u0000\u0000\u0172\u0174\u0005\u000f\u0000\u0000\u0173\u0175\u00036\u001b"+
+    "\u0000\u0174\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000"+
+    "\u0000\u0175\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005\u001c\u0000"+
+    "\u0000\u0177\u0179\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000"+
+    "\u0000\u0178\u0179\u0001\u0000\u0000\u0000\u01795\u0001\u0000\u0000\u0000"+
+    "\u017a\u017f\u00038\u001c\u0000\u017b\u017c\u0005!\u0000\u0000\u017c\u017e"+
+    "\u00038\u001c\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017e\u0181\u0001"+
+    "\u0000\u0000\u0000\u017f\u017d\u0001\u0000\u0000\u0000\u017f\u0180\u0001"+
+    "\u0000\u0000\u0000\u01807\u0001\u0000\u0000\u0000\u0181\u017f\u0001\u0000"+
+    "\u0000\u0000\u0182\u0185\u0003 \u0010\u0000\u0183\u0184\u0005\u0010\u0000"+
+    "\u0000\u0184\u0186\u0003\n\u0005\u0000\u0185\u0183\u0001\u0000\u0000\u0000"+
+    "\u0185\u0186\u0001\u0000\u0000\u0000\u01869\u0001\u0000\u0000\u0000\u0187"+
+    "\u018c\u0003H$\u0000\u0188\u0189\u0005#\u0000\u0000\u0189\u018b\u0003"+
+    "H$\u0000\u018a\u0188\u0001\u0000\u0000\u0000\u018b\u018e\u0001\u0000\u0000"+
+    "\u0000\u018c\u018a\u0001\u0000\u0000\u0000\u018c\u018d\u0001\u0000\u0000"+
+    "\u0000\u018d;\u0001\u0000\u0000\u0000\u018e\u018c\u0001\u0000\u0000\u0000"+
+    "\u018f\u0194\u0003B!\u0000\u0190\u0191\u0005#\u0000\u0000\u0191\u0193"+
+    "\u0003B!\u0000\u0192\u0190\u0001\u0000\u0000\u0000\u0193\u0196\u0001\u0000"+
+    "\u0000\u0000\u0194\u0192\u0001\u0000\u0000\u0000\u0194\u0195\u0001\u0000"+
+    "\u0000\u0000\u0195=\u0001\u0000\u0000\u0000\u0196\u0194\u0001\u0000\u0000"+
+    "\u0000\u0197\u019c\u0003<\u001e\u0000\u0198\u0199\u0005!\u0000\u0000\u0199"+
+    "\u019b\u0003<\u001e\u0000\u019a\u0198\u0001\u0000\u0000\u0000\u019b\u019e"+
+    "\u0001\u0000\u0000\u0000\u019c\u019a\u0001\u0000\u0000\u0000\u019c\u019d"+
+    "\u0001\u0000\u0000\u0000\u019d?\u0001\u0000\u0000\u0000\u019e\u019c\u0001"+
+    "\u0000\u0000\u0000\u019f\u01a0\u0007\u0003\u0000\u0000\u01a0A\u0001\u0000"+
+    "\u0000\u0000\u01a1\u01a5\u0005P\u0000\u0000\u01a2\u01a3\u0004!\n\u0000"+
+    "\u01a3\u01a5\u0003F#\u0000\u01a4\u01a1\u0001\u0000\u0000\u0000\u01a4\u01a2"+
+    "\u0001\u0000\u0000\u0000\u01a5C\u0001\u0000\u0000\u0000\u01a6\u01d1\u0005"+
+    ",\u0000\u0000\u01a7\u01a8\u0003h4\u0000\u01a8\u01a9\u0005C\u0000\u0000"+
+    "\u01a9\u01d1\u0001\u0000\u0000\u0000\u01aa\u01d1\u0003f3\u0000\u01ab\u01d1"+
+    "\u0003h4\u0000\u01ac\u01d1\u0003b1\u0000\u01ad\u01d1\u0003F#\u0000\u01ae"+
+    "\u01d1\u0003j5\u0000\u01af\u01b0\u0005A\u0000\u0000\u01b0\u01b5\u0003"+
+    "d2\u0000\u01b1\u01b2\u0005!\u0000\u0000\u01b2\u01b4\u0003d2\u0000\u01b3"+
+    "\u01b1\u0001\u0000\u0000\u0000\u01b4\u01b7\u0001\u0000\u0000\u0000\u01b5"+
+    "\u01b3\u0001\u0000\u0000\u0000\u01b5\u01b6\u0001\u0000\u0000\u0000\u01b6"+
+    "\u01b8\u0001\u0000\u0000\u0000\u01b7\u01b5\u0001\u0000\u0000\u0000\u01b8"+
+    "\u01b9\u0005B\u0000\u0000\u01b9\u01d1\u0001\u0000\u0000\u0000\u01ba\u01bb"+
+    "\u0005A\u0000\u0000\u01bb\u01c0\u0003b1\u0000\u01bc\u01bd\u0005!\u0000"+
+    "\u0000\u01bd\u01bf\u0003b1\u0000\u01be\u01bc\u0001\u0000\u0000\u0000\u01bf"+
+    "\u01c2\u0001\u0000\u0000\u0000\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0"+
+    "\u01c1\u0001\u0000\u0000\u0000\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2"+
+    "\u01c0\u0001\u0000\u0000\u0000\u01c3\u01c4\u0005B\u0000\u0000\u01c4\u01d1"+
+    "\u0001\u0000\u0000\u0000\u01c5\u01c6\u0005A\u0000\u0000\u01c6\u01cb\u0003"+
+    "j5\u0000\u01c7\u01c8\u0005!\u0000\u0000\u01c8\u01ca\u0003j5\u0000\u01c9"+
+    "\u01c7\u0001\u0000\u0000\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb"+
+    "\u01c9\u0001\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc"+
+    "\u01ce\u0001\u0000\u0000\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce"+
+    "\u01cf\u0005B\u0000\u0000\u01cf\u01d1\u0001\u0000\u0000\u0000\u01d0\u01a6"+
+    "\u0001\u0000\u0000\u0000\u01d0\u01a7\u0001\u0000\u0000\u0000\u01d0\u01aa"+
+    "\u0001\u0000\u0000\u0000\u01d0\u01ab\u0001\u0000\u0000\u0000\u01d0\u01ac"+
+    "\u0001\u0000\u0000\u0000\u01d0\u01ad\u0001\u0000\u0000\u0000\u01d0\u01ae"+
+    "\u0001\u0000\u0000\u0000\u01d0\u01af\u0001\u0000\u0000\u0000\u01d0\u01ba"+
+    "\u0001\u0000\u0000\u0000\u01d0\u01c5\u0001\u0000\u0000\u0000\u01d1E\u0001"+
+    "\u0000\u0000\u0000\u01d2\u01d5\u0005/\u0000\u0000\u01d3\u01d5\u0005@\u0000"+
+    "\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d4\u01d3\u0001\u0000\u0000"+
+    "\u0000\u01d5G\u0001\u0000\u0000\u0000\u01d6\u01da\u0003@ \u0000\u01d7"+
+    "\u01d8\u0004$\u000b\u0000\u01d8\u01da\u0003F#\u0000\u01d9\u01d6\u0001"+
+    "\u0000\u0000\u0000\u01d9\u01d7\u0001\u0000\u0000\u0000\u01daI\u0001\u0000"+
+    "\u0000\u0000\u01db\u01dc\u0005\t\u0000\u0000\u01dc\u01dd\u0005\u001a\u0000"+
+    "\u0000\u01ddK\u0001\u0000\u0000\u0000\u01de\u01df\u0005\u000e\u0000\u0000"+
+    "\u01df\u01e4\u0003N\'\u0000\u01e0\u01e1\u0005!\u0000\u0000\u01e1\u01e3"+
+    "\u0003N\'\u0000\u01e2\u01e0\u0001\u0000\u0000\u0000\u01e3\u01e6\u0001"+
+    "\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e4\u01e5\u0001"+
+    "\u0000\u0000\u0000\u01e5M\u0001\u0000\u0000\u0000\u01e6\u01e4\u0001\u0000"+
+    "\u0000\u0000\u01e7\u01e9\u0003\n\u0005\u0000\u01e8\u01ea\u0007\u0004\u0000"+
+    "\u0000\u01e9\u01e8\u0001\u0000\u0000\u0000\u01e9\u01ea\u0001\u0000\u0000"+
+    "\u0000\u01ea\u01ed\u0001\u0000\u0000\u0000\u01eb\u01ec\u0005-\u0000\u0000"+
+    "\u01ec\u01ee\u0007\u0005\u0000\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+
+    "\u01ed\u01ee\u0001\u0000\u0000\u0000\u01eeO\u0001\u0000\u0000\u0000\u01ef"+
+    "\u01f0\u0005\b\u0000\u0000\u01f0\u01f1\u0003>\u001f\u0000\u01f1Q\u0001"+
+    "\u0000\u0000\u0000\u01f2\u01f3\u0005\u0002\u0000\u0000\u01f3\u01f4\u0003"+
+    ">\u001f\u0000\u01f4S\u0001\u0000\u0000\u0000\u01f5\u01f6\u0005\u000b\u0000"+
+    "\u0000\u01f6\u01fb\u0003V+\u0000\u01f7\u01f8\u0005!\u0000\u0000\u01f8"+
+    "\u01fa\u0003V+\u0000\u01f9\u01f7\u0001\u0000\u0000\u0000\u01fa\u01fd\u0001"+
+    "\u0000\u0000\u0000\u01fb\u01f9\u0001\u0000\u0000\u0000\u01fb\u01fc\u0001"+
+    "\u0000\u0000\u0000\u01fcU\u0001\u0000\u0000\u0000\u01fd\u01fb\u0001\u0000"+
+    "\u0000\u0000\u01fe\u01ff\u0003<\u001e\u0000\u01ff\u0200\u0005T\u0000\u0000"+
+    "\u0200\u0201\u0003<\u001e\u0000\u0201W\u0001\u0000\u0000\u0000\u0202\u0203"+
+    "\u0005\u0001\u0000\u0000\u0203\u0204\u0003\u0014\n\u0000\u0204\u0206\u0003"+
+    "j5\u0000\u0205\u0207\u0003^/\u0000\u0206\u0205\u0001\u0000\u0000\u0000"+
+    "\u0206\u0207\u0001\u0000\u0000\u0000\u0207Y\u0001\u0000\u0000\u0000\u0208"+
+    "\u0209\u0005\u0007\u0000\u0000\u0209\u020a\u0003\u0014\n\u0000\u020a\u020b"+
+    "\u0003j5\u0000\u020b[\u0001\u0000\u0000\u0000\u020c\u020d\u0005\n\u0000"+
+    "\u0000\u020d\u020e\u0003:\u001d\u0000\u020e]\u0001\u0000\u0000\u0000\u020f"+
+    "\u0214\u0003`0\u0000\u0210\u0211\u0005!\u0000\u0000\u0211\u0213\u0003"+
+    "`0\u0000\u0212\u0210\u0001\u0000\u0000\u0000\u0213\u0216\u0001\u0000\u0000"+
+    "\u0000\u0214\u0212\u0001\u0000\u0000\u0000\u0214\u0215\u0001\u0000\u0000"+
+    "\u0000\u0215_\u0001\u0000\u0000\u0000\u0216\u0214\u0001\u0000\u0000\u0000"+
+    "\u0217\u0218\u0003@ \u0000\u0218\u0219\u0005\u001f\u0000\u0000\u0219\u021a"+
+    "\u0003D\"\u0000\u021aa\u0001\u0000\u0000\u0000\u021b\u021c\u0007\u0006"+
+    "\u0000\u0000\u021cc\u0001\u0000\u0000\u0000\u021d\u0220\u0003f3\u0000"+
+    "\u021e\u0220\u0003h4\u0000\u021f\u021d\u0001\u0000\u0000\u0000\u021f\u021e"+
+    "\u0001\u0000\u0000\u0000\u0220e\u0001\u0000\u0000\u0000\u0221\u0223\u0007"+
+    "\u0000\u0000\u0000\u0222\u0221\u0001\u0000\u0000\u0000\u0222\u0223\u0001"+
+    "\u0000\u0000\u0000\u0223\u0224\u0001\u0000\u0000\u0000\u0224\u0225\u0005"+
+    "\u001b\u0000\u0000\u0225g\u0001\u0000\u0000\u0000\u0226\u0228\u0007\u0000"+
+    "\u0000\u0000\u0227\u0226\u0001\u0000\u0000\u0000\u0227\u0228\u0001\u0000"+
+    "\u0000\u0000\u0228\u0229\u0001\u0000\u0000\u0000\u0229\u022a\u0005\u001a"+
+    "\u0000\u0000\u022ai\u0001\u0000\u0000\u0000\u022b\u022c\u0005\u0019\u0000"+
+    "\u0000\u022ck\u0001\u0000\u0000\u0000\u022d\u022e\u0007\u0007\u0000\u0000"+
+    "\u022em\u0001\u0000\u0000\u0000\u022f\u0230\u0005\u0005\u0000\u0000\u0230"+
+    "\u0231\u0003p8\u0000\u0231o\u0001\u0000\u0000\u0000\u0232\u0233\u0005"+
+    "A\u0000\u0000\u0233\u0234\u0003\u0002\u0001\u0000\u0234\u0235\u0005B\u0000"+
+    "\u0000\u0235q\u0001\u0000\u0000\u0000\u0236\u0237\u0005\r\u0000\u0000"+
+    "\u0237\u0238\u0005d\u0000\u0000\u0238s\u0001\u0000\u0000\u0000\u0239\u023a"+
+    "\u0005\u0003\u0000\u0000\u023a\u023d\u0005Z\u0000\u0000\u023b\u023c\u0005"+
+    "X\u0000\u0000\u023c\u023e\u0003<\u001e\u0000\u023d\u023b\u0001\u0000\u0000"+
+    "\u0000\u023d\u023e\u0001\u0000\u0000\u0000\u023e\u0248\u0001\u0000\u0000"+
+    "\u0000\u023f\u0240\u0005Y\u0000\u0000\u0240\u0245\u0003v;\u0000\u0241"+
+    "\u0242\u0005!\u0000\u0000\u0242\u0244\u0003v;\u0000\u0243\u0241\u0001"+
+    "\u0000\u0000\u0000\u0244\u0247\u0001\u0000\u0000\u0000\u0245\u0243\u0001"+
+    "\u0000\u0000\u0000\u0245\u0246\u0001\u0000\u0000\u0000\u0246\u0249\u0001"+
+    "\u0000\u0000\u0000\u0247\u0245\u0001\u0000\u0000\u0000\u0248\u023f\u0001"+
+    "\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249u\u0001\u0000"+
+    "\u0000\u0000\u024a\u024b\u0003<\u001e\u0000\u024b\u024c\u0005\u001f\u0000"+
+    "\u0000\u024c\u024e\u0001\u0000\u0000\u0000\u024d\u024a\u0001\u0000\u0000"+
+    "\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f\u0001\u0000\u0000"+
+    "\u0000\u024f\u0250\u0003<\u001e\u0000\u0250w\u0001\u0000\u0000\u0000\u0251"+
+    "\u0252\u0005\u0012\u0000\u0000\u0252\u0253\u0003$\u0012\u0000\u0253\u0254"+
+    "\u0005X\u0000\u0000\u0254\u0255\u0003>\u001f\u0000\u0255y\u0001\u0000"+
+    "\u0000\u0000\u0256\u0257\u0005\u0011\u0000\u0000\u0257\u025a\u00036\u001b"+
+    "\u0000\u0258\u0259\u0005\u001c\u0000\u0000\u0259\u025b\u0003\u001e\u000f"+
+    "\u0000\u025a\u0258\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000"+
+    "\u0000\u025b{\u0001\u0000\u0000\u0000;\u0087\u0090\u00a2\u00ae\u00b7\u00bf"+
+    "\u00c5\u00cd\u00cf\u00d4\u00db\u00e0\u00eb\u00f1\u00f9\u00fb\u0106\u010d"+
+    "\u0118\u011b\u0121\u012d\u0133\u013d\u0141\u0146\u0150\u0158\u0165\u0169"+
+    "\u016d\u0174\u0178\u017f\u0185\u018c\u0194\u019c\u01a4\u01b5\u01c0\u01cb"+
+    "\u01d0\u01d4\u01d9\u01e4\u01e9\u01ed\u01fb\u0206\u0214\u021f\u0222\u0227"+
+    "\u023d\u0245\u0248\u024d\u025a";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
index 027281d44b2dc..556a97657635a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
@@ -332,6 +332,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterFunctionName(EsqlBaseParser.FunctionNameContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { }
   /**
    * {@inheritDoc}
    *
@@ -500,6 +512,30 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterAggFields(EsqlBaseParser.AggFieldsContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitAggFields(EsqlBaseParser.AggFieldsContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterAggField(EsqlBaseParser.AggFieldContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitAggField(EsqlBaseParser.AggFieldContext ctx) { }
   /**
    * {@inheritDoc}
    *
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
index 463414ab67ea2..56b6999615f50 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
@@ -202,6 +202,13 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -300,6 +307,20 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitAggFields(EsqlBaseParser.AggFieldsContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitAggField(EsqlBaseParser.AggFieldContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
index 1747ff001e162..cf658c4a73141 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
@@ -313,6 +313,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#functionName}.
+   * @param ctx the parse tree
+   */
+  void enterFunctionName(EsqlBaseParser.FunctionNameContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#functionName}.
+   * @param ctx the parse tree
+   */
+  void exitFunctionName(EsqlBaseParser.FunctionNameContext ctx);
   /**
    * Enter a parse tree produced by the {@code toDataType}
    * labeled alternative in {@link EsqlBaseParser#dataType}.
@@ -455,6 +465,26 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitStatsCommand(EsqlBaseParser.StatsCommandContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#aggFields}.
+   * @param ctx the parse tree
+   */
+  void enterAggFields(EsqlBaseParser.AggFieldsContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#aggFields}.
+   * @param ctx the parse tree
+   */
+  void exitAggFields(EsqlBaseParser.AggFieldsContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#aggField}.
+   * @param ctx the parse tree
+   */
+  void enterAggField(EsqlBaseParser.AggFieldContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#aggField}.
+   * @param ctx the parse tree
+   */
+  void exitAggField(EsqlBaseParser.AggFieldContext ctx);
   /**
    * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedName}.
    * @param ctx the parse tree
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
index b7411d0f99c09..86c1d1aafc33a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
@@ -193,6 +193,12 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#functionName}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitFunctionName(EsqlBaseParser.FunctionNameContext ctx);
   /**
    * Visit a parse tree produced by the {@code toDataType}
    * labeled alternative in {@link EsqlBaseParser#dataType}.
@@ -278,6 +284,18 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#aggFields}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitAggFields(EsqlBaseParser.AggFieldsContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#aggField}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitAggField(EsqlBaseParser.AggFieldContext ctx);
   /**
    * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedName}.
    * @param ctx the parse tree
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
index 5696ccea188b1..620a25e0170ea 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
@@ -98,10 +98,8 @@ private class PostProcessor extends EsqlBaseParserBaseListener {
         @Override
         public void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) {
             // TODO remove this at some point
-            EsqlBaseParser.IdentifierOrParameterContext identifierOrParameter = ctx.identifierOrParameter();
-            EsqlBaseParser.IdentifierContext idCtx = identifierOrParameter.identifier();
-            String functionName = idCtx != null ? idCtx.getText() : identifierOrParameter.parameter().getText();
-            if ("is_null".equalsIgnoreCase(functionName)) {
+            EsqlBaseParser.FunctionNameContext identifier = ctx.functionName();
+            if (identifier.getText().equalsIgnoreCase("is_null")) {
                 throw new ParsingException(
                     source(ctx),
                     "is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead"
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
index 50cf4bc998679..5903d725bf9c2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
@@ -44,6 +45,7 @@
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.expression.function.FunctionResolutionStrategy;
 import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
@@ -439,7 +441,7 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat
             // use the fast run variant
             result = new UnresolvedNamePattern(
                 src,
-                new CharacterRunAutomaton(Operations.concatenate(list)),
+                new CharacterRunAutomaton(Operations.determinize(Operations.concatenate(list), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)),
                 patternString.toString(),
                 nameString.toString()
             );
@@ -593,13 +595,7 @@ public UnresolvedAttribute visitDereference(EsqlBaseParser.DereferenceContext ct
 
     @Override
     public Expression visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) {
-        EsqlBaseParser.IdentifierOrParameterContext identifierOrParameter = ctx.identifierOrParameter();
-        String name;
-        if (identifierOrParameter.identifier() != null) {
-            name = visitIdentifier(identifierOrParameter.identifier());
-        } else {
-            name = unresolvedAttributeNameInParam(identifierOrParameter.parameter(), expression(identifierOrParameter.parameter()));
-        }
+        String name = visitFunctionName(ctx.functionName());
         List<Expression> args = expressions(ctx.booleanExpression());
         if ("is_null".equals(EsqlFunctionRegistry.normalizeName(name))) {
             throw new ParsingException(
@@ -616,6 +612,23 @@ public Expression visitFunctionExpression(EsqlBaseParser.FunctionExpressionConte
         return new UnresolvedFunction(source(ctx), name, FunctionResolutionStrategy.DEFAULT, args);
     }
 
+    @Override
+    public String visitFunctionName(EsqlBaseParser.FunctionNameContext ctx) {
+        if (ctx.MATCH() != null) {
+            return ctx.MATCH().getText();
+        }
+        return visitIdentifierOrParameter(ctx.identifierOrParameter());
+    }
+
+    @Override
+    public String visitIdentifierOrParameter(EsqlBaseParser.IdentifierOrParameterContext ctx) {
+        if (ctx.identifier() != null) {
+            return visitIdentifier(ctx.identifier());
+        }
+
+        return unresolvedAttributeNameInParam(ctx.parameter(), expression(ctx.parameter()));
+    }
+
     @Override
     public Expression visitInlineCast(EsqlBaseParser.InlineCastContext ctx) {
         Source source = source(ctx);
@@ -731,9 +744,12 @@ private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternConte
 
     @Override
     public Alias visitField(EsqlBaseParser.FieldContext ctx) {
+        return visitField(ctx, source(ctx));
+    }
+
+    private Alias visitField(EsqlBaseParser.FieldContext ctx, Source source) {
         UnresolvedAttribute id = visitQualifiedName(ctx.qualifiedName());
         Expression value = expression(ctx.booleanExpression());
-        var source = source(ctx);
         String name = id == null ? source.text() : id.name();
         return new Alias(source, name, value);
     }
@@ -743,6 +759,36 @@ public List<Alias> visitFields(EsqlBaseParser.FieldsContext ctx) {
         return ctx != null ? visitList(this, ctx.field(), Alias.class) : new ArrayList<>();
     }
 
+    @Override
+    public NamedExpression visitAggField(EsqlBaseParser.AggFieldContext ctx) {
+        Source source = source(ctx);
+        Alias field = visitField(ctx.field(), source);
+        var filterExpression = ctx.booleanExpression();
+
+        if (filterExpression != null) {
+            Expression condition = expression(filterExpression);
+            Expression child = field.child();
+            // basic check as the filter can be specified only on a function (should be an aggregate but we can't determine that yet)
+            if (field.child().anyMatch(Function.class::isInstance)) {
+                field = field.replaceChild(new FilteredExpression(field.source(), child, condition));
+            }
+            // allow condition only per aggregated function
+            else {
+                throw new ParsingException(
+                    condition.source(),
+                    "WHERE clause allowed only for aggregate functions [{}]",
+                    field.sourceText()
+                );
+            }
+        }
+        return field;
+    }
+
+    @Override
+    public List<Alias> visitAggFields(EsqlBaseParser.AggFieldsContext ctx) {
+        return ctx != null ? visitList(this, ctx.aggField(), Alias.class) : new ArrayList<>();
+    }
+
     /**
      * Similar to {@link #visitFields(EsqlBaseParser.FieldsContext)} however avoids wrapping the expression
      * into an Alias.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java
index 1872fa6e8f1f0..ae2379318474b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java
@@ -21,6 +21,7 @@
 import java.util.List;
 
 import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR;
+import static org.elasticsearch.transport.RemoteClusterAware.isRemoteIndexName;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.EXCLUSION;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
 import static org.elasticsearch.xpack.esql.parser.ParserUtils.source;
@@ -61,11 +62,14 @@ public String visitIndexPattern(List<EsqlBaseParser.IndexPatternContext> ctx) {
         Holder<Boolean> hasSeenStar = new Holder<>(false);
         ctx.forEach(c -> {
             String indexPattern = visitIndexString(c.indexString());
-            hasSeenStar.set(indexPattern.contains(WILDCARD) || hasSeenStar.get());
-            validateIndexPattern(indexPattern, c, hasSeenStar.get());
-            patterns.add(
-                c.clusterString() != null ? c.clusterString().getText() + REMOTE_CLUSTER_INDEX_SEPARATOR + indexPattern : indexPattern
-            );
+            String clusterString = c.clusterString() != null ? c.clusterString().getText() : null;
+            // skip validating index on remote cluster, because the behavior of remote cluster is not consistent with local cluster
+            // For example, invalid#index is an invalid index name, however FROM *:invalid#index does not return an error
+            if (clusterString == null) {
+                hasSeenStar.set(indexPattern.contains(WILDCARD) || hasSeenStar.get());
+                validateIndexPattern(indexPattern, c, hasSeenStar.get());
+            }
+            patterns.add(clusterString != null ? clusterString + REMOTE_CLUSTER_INDEX_SEPARATOR + indexPattern : indexPattern);
         });
         return Strings.collectionToDelimitedString(patterns, ",");
     }
@@ -75,6 +79,9 @@ private static void validateIndexPattern(String indexPattern, EsqlBaseParser.Ind
         String[] indices = indexPattern.split(",");
         boolean hasExclusion = false;
         for (String index : indices) {
+            if (isRemoteIndexName(index)) { // skip the validation if there is remote cluster
+                continue;
+            }
             hasSeenStar = index.contains(WILDCARD) || hasSeenStar;
             index = index.replace(WILDCARD, "").strip();
             if (index.isBlank()) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
index c90c3cba4ef24..dc913cd2f14f4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
@@ -298,13 +298,12 @@ public PlanFactory visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) {
         return input -> new Aggregate(source(ctx), input, Aggregate.AggregateType.STANDARD, stats.groupings, stats.aggregates);
     }
 
-    private record Stats(List<Expression> groupings, List<? extends NamedExpression> aggregates) {
+    private record Stats(List<Expression> groupings, List<? extends NamedExpression> aggregates) {}
 
-    }
-
-    private Stats stats(Source source, EsqlBaseParser.FieldsContext groupingsCtx, EsqlBaseParser.FieldsContext aggregatesCtx) {
+    private Stats stats(Source source, EsqlBaseParser.FieldsContext groupingsCtx, EsqlBaseParser.AggFieldsContext aggregatesCtx) {
         List<NamedExpression> groupings = visitGrouping(groupingsCtx);
-        List<NamedExpression> aggregates = new ArrayList<>(visitFields(aggregatesCtx));
+        List<NamedExpression> aggregates = new ArrayList<>(visitAggFields(aggregatesCtx));
+
         if (aggregates.isEmpty() && groupings.isEmpty()) {
             throw new ParsingException(source, "At least one aggregation or grouping expression required in [{}]", source.text());
         }
@@ -341,9 +340,11 @@ public PlanFactory visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandCont
         if (false == EsqlPlugin.INLINESTATS_FEATURE_FLAG.isEnabled()) {
             throw new ParsingException(source(ctx), "INLINESTATS command currently requires a snapshot build");
         }
-        List<NamedExpression> aggregates = new ArrayList<>(visitFields(ctx.stats));
+        List<Alias> aggFields = visitAggFields(ctx.stats);
+        List<NamedExpression> aggregates = new ArrayList<>(aggFields);
         List<NamedExpression> groupings = visitGrouping(ctx.grouping);
         aggregates.addAll(groupings);
+        // TODO: add support for filters
         return input -> new InlineStats(source(ctx), input, new ArrayList<>(groupings), aggregates);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
index 8445c8236c45a..e1632db4f79a2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
+import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -40,7 +41,7 @@ public enum AggregateType {
         METRICS;
 
         static void writeType(StreamOutput out, AggregateType type) throws IOException {
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_AGGREGATE_TYPE)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 out.writeString(type.name());
             } else if (type != STANDARD) {
                 throw new IllegalStateException("cluster is not ready to support aggregate type [" + type + "]");
@@ -48,7 +49,7 @@ static void writeType(StreamOutput out, AggregateType type) throws IOException {
         }
 
         static AggregateType readType(StreamInput in) throws IOException {
-            if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_AGGREGATE_TYPE)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
                 return AggregateType.valueOf(in.readString());
             } else {
                 return STANDARD;
@@ -59,6 +60,7 @@ static AggregateType readType(StreamInput in) throws IOException {
     private final AggregateType aggregateType;
     private final List<Expression> groupings;
     private final List<? extends NamedExpression> aggregates;
+
     private List<Attribute> lazyOutput;
 
     public Aggregate(
@@ -156,7 +158,13 @@ protected AttributeSet computeReferences() {
     }
 
     public static AttributeSet computeReferences(List<? extends NamedExpression> aggregates, List<? extends Expression> groupings) {
-        return Expressions.references(groupings).combine(Expressions.references(aggregates));
+        AttributeSet result = Expressions.references(groupings).combine(Expressions.references(aggregates));
+        for (Expression grouping : groupings) {
+            if (grouping instanceof Alias) {
+                result.remove(((Alias) grouping).toAttribute());
+            }
+        }
+        return result;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
index 0043362f23b87..eb72009638396 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
@@ -87,7 +87,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     private static boolean supportingEsSourceOptions(TransportVersion version) {
-        return version.onOrAfter(TransportVersions.V_8_14_0) && version.before(TransportVersions.ESQL_REMOVE_ES_SOURCE_OPTIONS);
+        return version.between(TransportVersions.V_8_14_0, TransportVersions.V_8_15_0);
     }
 
     @Override
@@ -112,7 +112,12 @@ private static List<Attribute> flatten(Source source, Map<String, EsField> mappi
             EsField t = entry.getValue();
 
             if (t != null) {
-                FieldAttribute f = new FieldAttribute(source, parent, parent != null ? parent.name() + "." + name : name, t);
+                FieldAttribute f = new FieldAttribute(
+                    source,
+                    parent != null ? parent.name() : null,
+                    parent != null ? parent.name() + "." + name : name,
+                    t
+                );
                 list.add(f);
                 // object or nested
                 if (t.getProperties().isEmpty() == false) {
@@ -180,7 +185,7 @@ public String nodeString() {
     }
 
     public static IndexMode readIndexMode(StreamInput in) throws IOException {
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             return IndexMode.fromString(in.readString());
         } else {
             return IndexMode.STANDARD;
@@ -188,7 +193,7 @@ public static IndexMode readIndexMode(StreamInput in) throws IOException {
     }
 
     public static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeString(indexMode.getName());
         } else if (indexMode != IndexMode.STANDARD) {
             throw new IllegalStateException("not ready to support index mode [" + indexMode + "]");
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java
index 0e71963e29270..94a9246a56f83 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java
@@ -10,10 +10,12 @@
 import org.elasticsearch.compute.aggregation.Aggregator;
 import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.AggregatorMode;
+import org.elasticsearch.compute.aggregation.FilteredAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.GroupingAggregator;
 import org.elasticsearch.compute.aggregation.blockhash.BlockHash;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.operator.AggregationOperator;
+import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.HashAggregationOperator.HashAggregationOperatorFactory;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
@@ -24,6 +26,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
@@ -231,11 +234,14 @@ private void aggregatesToFactory(
         boolean grouping,
         Consumer<AggFunctionSupplierContext> consumer
     ) {
+        // extract filtering channels - and wrap the aggregation with the new evaluator expression only during the init phase
         for (NamedExpression ne : aggregates) {
+            // a filter can only appear on aggregate function, not on the grouping columns
+
             if (ne instanceof Alias alias) {
                 var child = alias.child();
                 if (child instanceof AggregateFunction aggregateFunction) {
-                    List<? extends NamedExpression> sourceAttr;
+                    List<NamedExpression> sourceAttr = new ArrayList<>();
 
                     if (mode == AggregatorMode.INITIAL) {
                         // TODO: this needs to be made more reliable - use casting to blow up when dealing with expressions (e+1)
@@ -251,19 +257,22 @@ private void aggregatesToFactory(
                                 );
                             }
                         } else {
-                            sourceAttr = aggregateFunction.inputExpressions().stream().map(e -> {
-                                Attribute attr = Expressions.attribute(e);
+                            // extra dependencies like TS ones (that require a timestamp)
+                            for (Expression input : aggregateFunction.references()) {
+                                Attribute attr = Expressions.attribute(input);
                                 if (attr == null) {
                                     throw new EsqlIllegalArgumentException(
                                         "Cannot work with target field [{}] for agg [{}]",
-                                        e.sourceText(),
+                                        input.sourceText(),
                                         aggregateFunction.sourceText()
                                     );
                                 }
-                                return attr;
-                            }).toList();
+                                sourceAttr.add(attr);
+                            }
                         }
-                    } else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) {
+                    }
+                    // coordinator/exchange phase
+                    else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) {
                         if (grouping) {
                             sourceAttr = aggregateMapper.mapGrouping(aggregateFunction);
                         } else {
@@ -274,16 +283,27 @@ private void aggregatesToFactory(
                     }
                     List<Integer> inputChannels = sourceAttr.stream().map(attr -> layout.get(attr.id()).channel()).toList();
                     assert inputChannels.stream().allMatch(i -> i >= 0) : inputChannels;
-                    if (aggregateFunction instanceof ToAggregator agg) {
-                        consumer.accept(new AggFunctionSupplierContext(agg.supplier(inputChannels), mode));
-                    } else {
-                        throw new EsqlIllegalArgumentException("aggregate functions must extend ToAggregator");
+
+                    AggregatorFunctionSupplier aggSupplier = supplier(aggregateFunction, inputChannels);
+
+                    // apply the filter only in the initial phase - as the rest of the data is already filtered
+                    if (aggregateFunction.hasFilter() && mode.isInputPartial() == false) {
+                        EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator(aggregateFunction.filter(), layout);
+                        aggSupplier = new FilteredAggregatorFunctionSupplier(aggSupplier, evalFactory);
                     }
+                    consumer.accept(new AggFunctionSupplierContext(aggSupplier, mode));
                 }
             }
         }
     }
 
+    private static AggregatorFunctionSupplier supplier(AggregateFunction aggregateFunction, List<Integer> inputChannels) {
+        if (aggregateFunction instanceof ToAggregator delegate) {
+            return delegate.supplier(inputChannels);
+        }
+        throw new EsqlIllegalArgumentException("aggregate functions must extend ToAggregator");
+    }
+
     private record GroupSpec(Integer channel, Attribute attribute) {
         BlockHash.GroupSpec toHashGroupSpec() {
             if (channel == null) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
index 13ce9ba77cc71..3e81c2a2c1101 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
@@ -98,39 +98,39 @@ private record AggDef(Class<?> aggClazz, String type, String extra, boolean grou
         .collect(Collectors.toUnmodifiableMap(aggDef -> aggDef, AggregateMapper::lookupIntermediateState));
 
     /** Cache of aggregates to intermediate expressions. */
-    private final HashMap<Expression, List<? extends NamedExpression>> cache;
+    private final HashMap<Expression, List<NamedExpression>> cache;
 
     AggregateMapper() {
         cache = new HashMap<>();
     }
 
-    public List<? extends NamedExpression> mapNonGrouping(List<? extends Expression> aggregates) {
+    public List<NamedExpression> mapNonGrouping(List<? extends Expression> aggregates) {
         return doMapping(aggregates, false);
     }
 
-    public List<? extends NamedExpression> mapNonGrouping(Expression aggregate) {
+    public List<NamedExpression> mapNonGrouping(Expression aggregate) {
         return map(aggregate, false).toList();
     }
 
-    public List<? extends NamedExpression> mapGrouping(List<? extends Expression> aggregates) {
+    public List<NamedExpression> mapGrouping(List<? extends Expression> aggregates) {
         return doMapping(aggregates, true);
     }
 
-    private List<? extends NamedExpression> doMapping(List<? extends Expression> aggregates, boolean grouping) {
+    private List<NamedExpression> doMapping(List<? extends Expression> aggregates, boolean grouping) {
         AttributeMap<NamedExpression> attrToExpressions = new AttributeMap<>();
         aggregates.stream().flatMap(agg -> map(agg, grouping)).forEach(ne -> attrToExpressions.put(ne.toAttribute(), ne));
         return attrToExpressions.values().stream().toList();
     }
 
-    public List<? extends NamedExpression> mapGrouping(Expression aggregate) {
+    public List<NamedExpression> mapGrouping(Expression aggregate) {
         return map(aggregate, true).toList();
     }
 
-    private Stream<? extends NamedExpression> map(Expression aggregate, boolean grouping) {
+    private Stream<NamedExpression> map(Expression aggregate, boolean grouping) {
         return cache.computeIfAbsent(Alias.unwrap(aggregate), aggKey -> computeEntryForAgg(aggKey, grouping)).stream();
     }
 
-    private static List<? extends NamedExpression> computeEntryForAgg(Expression aggregate, boolean grouping) {
+    private static List<NamedExpression> computeEntryForAgg(Expression aggregate, boolean grouping) {
         var aggDef = aggDefOrNull(aggregate, grouping);
         if (aggDef != null) {
             var is = getNonNull(aggDef);
@@ -297,25 +297,18 @@ private static String dataTypeToString(DataType type, Class<?> aggClass) {
         if (aggClass == Top.class && type.equals(DataType.IP)) {
             return "Ip";
         }
-        if (type.equals(DataType.BOOLEAN)) {
-            return "Boolean";
-        } else if (type.equals(DataType.INTEGER) || type.equals(DataType.COUNTER_INTEGER)) {
-            return "Int";
-        } else if (type.equals(DataType.LONG) || type.equals(DataType.DATETIME) || type.equals(DataType.COUNTER_LONG)) {
-            return "Long";
-        } else if (type.equals(DataType.DOUBLE) || type.equals(DataType.COUNTER_DOUBLE)) {
-            return "Double";
-        } else if (type.equals(DataType.KEYWORD)
-            || type.equals(DataType.IP)
-            || type.equals(DataType.VERSION)
-            || type.equals(DataType.TEXT)) {
-                return "BytesRef";
-            } else if (type.equals(GEO_POINT)) {
-                return "GeoPoint";
-            } else if (type.equals(CARTESIAN_POINT)) {
-                return "CartesianPoint";
-            } else {
+
+        return switch (type) {
+            case DataType.BOOLEAN -> "Boolean";
+            case DataType.INTEGER, DataType.COUNTER_INTEGER -> "Int";
+            case DataType.LONG, DataType.DATETIME, DataType.COUNTER_LONG, DataType.DATE_NANOS -> "Long";
+            case DataType.DOUBLE, DataType.COUNTER_DOUBLE -> "Double";
+            case DataType.KEYWORD, DataType.IP, DataType.VERSION, DataType.TEXT -> "BytesRef";
+            case GEO_POINT -> "GeoPoint";
+            case CARTESIAN_POINT -> "CartesianPoint";
+            case SEMANTIC_TEXT, UNSUPPORTED, NULL, UNSIGNED_LONG, SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, SOURCE, DATE_PERIOD,
+                TIME_DURATION, CARTESIAN_SHAPE, GEO_SHAPE, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG ->
                 throw new EsqlIllegalArgumentException("illegal agg type: " + type.typeName());
-            }
+        };
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
index 18aa2628fdc7c..2c8604a7c4a80 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
@@ -24,15 +24,18 @@
 import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators;
 import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler;
 import org.elasticsearch.xpack.esql.core.querydsl.query.MatchAll;
+import org.elasticsearch.xpack.esql.core.querydsl.query.MatchQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
+import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Check;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils;
@@ -55,6 +58,7 @@
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
@@ -85,17 +89,11 @@ public final class EsqlExpressionTranslators {
         new ExpressionTranslators.StringQueries(),
         new ExpressionTranslators.Matches(),
         new ExpressionTranslators.MultiMatches(),
-        new FullTextFunctions(),
+        new MatchFunctionTranslator(),
+        new QueryStringFunctionTranslator(),
         new Scalars()
     );
 
-    public static class FullTextFunctions extends ExpressionTranslator<FullTextFunction> {
-        @Override
-        protected Query asQuery(FullTextFunction fullTextFunction, TranslatorHandler handler) {
-            return fullTextFunction.asQuery();
-        }
-    }
-
     public static Query toQuery(Expression e, TranslatorHandler handler) {
         Query translation = null;
         for (ExpressionTranslator<?> translator : QUERY_TRANSLATORS) {
@@ -528,4 +526,18 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) {
             );
         }
     }
+
+    public static class MatchFunctionTranslator extends ExpressionTranslator<Match> {
+        @Override
+        protected Query asQuery(Match match, TranslatorHandler handler) {
+            return new MatchQuery(match.source(), ((FieldAttribute) match.field()).name(), match.queryAsText());
+        }
+    }
+
+    public static class QueryStringFunctionTranslator extends ExpressionTranslator<QueryString> {
+        @Override
+        protected Query asQuery(QueryString queryString, TranslatorHandler handler) {
+            return new QueryStringQuery(queryString.source(), queryString.queryAsText(), Map.of(), null);
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
index b28c80211c649..dc732258d9fa5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
@@ -349,7 +349,7 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte
             elementTypes[channel] = PlannerUtils.toElementType(inverse.get(channel).type());
             encoders[channel] = switch (inverse.get(channel).type()) {
                 case IP -> TopNEncoder.IP;
-                case TEXT, KEYWORD -> TopNEncoder.UTF8;
+                case TEXT, KEYWORD, SEMANTIC_TEXT -> TopNEncoder.UTF8;
                 case VERSION -> TopNEncoder.VERSION;
                 case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_NANOS, DATE_PERIOD, TIME_DURATION,
                     OBJECT, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java
index e571be54692c4..152c492a34433 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java
@@ -52,8 +52,10 @@
 import org.elasticsearch.xpack.esql.plan.physical.RowExec;
 import org.elasticsearch.xpack.esql.plan.physical.ShowExec;
 import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
+import org.elasticsearch.xpack.esql.plan.physical.UnaryExec;
 
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * <p>This class is part of the planner</p>
@@ -104,6 +106,46 @@ public PhysicalPlan map(LogicalPlan p) {
         //
         // Unary Plan
         //
+        if (localMode == false && p instanceof Enrich enrich && enrich.mode() == Enrich.Mode.REMOTE) {
+            // When we have remote enrich, we want to put it under FragmentExec, so it would be executed remotely.
+            // We're only going to do it on the coordinator node.
+            // The way we're going to do it is as follows:
+            // 1. Locate FragmentExec in the tree. If we have no FragmentExec, we won't do anything.
+            // 2. Put this Enrich under it, removing everything that was below it previously.
+            // 3. Above FragmentExec, we should deal with pipeline breakers, since pipeline ops already are supposed to go under
+            // FragmentExec.
+            // 4. Aggregates can't appear here since the plan should have errored out if we have aggregate inside remote Enrich.
+            // 5. So we should be keeping: LimitExec, ExchangeExec, OrderExec, TopNExec (actually OrderExec probably can't happen anyway).
+
+            var child = map(enrich.child());
+            AtomicBoolean hasFragment = new AtomicBoolean(false);
+
+            var childTransformed = child.transformUp((f) -> {
+                // Once we reached FragmentExec, we stuff our Enrich under it
+                if (f instanceof FragmentExec) {
+                    hasFragment.set(true);
+                    return new FragmentExec(p);
+                }
+                if (f instanceof EnrichExec enrichExec) {
+                    // It can only be ANY because COORDINATOR would have errored out earlier, and REMOTE should be under FragmentExec
+                    assert enrichExec.mode() == Enrich.Mode.ANY : "enrich must be in ANY mode here";
+                    return enrichExec.child();
+                }
+                if (f instanceof UnaryExec unaryExec) {
+                    if (f instanceof LimitExec || f instanceof ExchangeExec || f instanceof OrderExec || f instanceof TopNExec) {
+                        return f;
+                    } else {
+                        return unaryExec.child();
+                    }
+                }
+                // Currently, it's either UnaryExec or LeafExec. Leaf will either resolve to FragmentExec or we'll ignore it.
+                return f;
+            });
+
+            if (hasFragment.get()) {
+                return childTransformed;
+            }
+        }
 
         if (p instanceof UnaryPlan ua) {
             var child = map(ua.child());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
index 7beed64dda8cb..7868984d6b6e2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
@@ -247,7 +247,7 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field
             case INTEGER, COUNTER_INTEGER -> ElementType.INT;
             case DOUBLE, COUNTER_DOUBLE -> ElementType.DOUBLE;
             // unsupported fields are passed through as a BytesRef
-            case KEYWORD, TEXT, IP, SOURCE, VERSION, UNSUPPORTED -> ElementType.BYTES_REF;
+            case KEYWORD, TEXT, IP, SOURCE, VERSION, SEMANTIC_TEXT, UNSUPPORTED -> ElementType.BYTES_REF;
             case NULL -> ElementType.NULL;
             case BOOLEAN -> ElementType.BOOLEAN;
             case DOC_DATA_TYPE -> ElementType.DOC;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java
index d8fc4da070767..49af4a593e6e5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java
@@ -47,7 +47,6 @@ final class ComputeListener implements Releasable {
     private final List<DriverProfile> collectedProfiles;
     private final ResponseHeadersCollector responseHeaders;
     private final EsqlExecutionInfo esqlExecutionInfo;
-    private final long queryStartTimeNanos;
     // clusterAlias indicating where this ComputeListener is running
     // used by the top level ComputeListener in ComputeService on both local and remote clusters
     private final String whereRunning;
@@ -61,7 +60,7 @@ public static ComputeListener create(
         CancellableTask task,
         ActionListener<ComputeResponse> delegate
     ) {
-        return new ComputeListener(transportService, task, null, null, -1, delegate);
+        return new ComputeListener(transportService, task, null, null, delegate);
     }
 
     /**
@@ -75,7 +74,6 @@ public static ComputeListener create(
      * @param transportService
      * @param task
      * @param executionInfo {@link EsqlExecutionInfo} to capture execution metadata
-     * @param queryStartTimeNanos Start time of the ES|QL query (stored in {@link org.elasticsearch.xpack.esql.session.Configuration})
      * @param delegate
      */
     public static ComputeListener create(
@@ -83,10 +81,9 @@ public static ComputeListener create(
         TransportService transportService,
         CancellableTask task,
         EsqlExecutionInfo executionInfo,
-        long queryStartTimeNanos,
         ActionListener<ComputeResponse> delegate
     ) {
-        return new ComputeListener(transportService, task, clusterAlias, executionInfo, queryStartTimeNanos, delegate);
+        return new ComputeListener(transportService, task, clusterAlias, executionInfo, delegate);
     }
 
     private ComputeListener(
@@ -94,7 +91,6 @@ private ComputeListener(
         CancellableTask task,
         String clusterAlias,
         EsqlExecutionInfo executionInfo,
-        long queryStartTimeNanos,
         ActionListener<ComputeResponse> delegate
     ) {
         this.transportService = transportService;
@@ -102,7 +98,6 @@ private ComputeListener(
         this.responseHeaders = new ResponseHeadersCollector(transportService.getThreadPool().getThreadContext());
         this.collectedProfiles = Collections.synchronizedList(new ArrayList<>());
         this.esqlExecutionInfo = executionInfo;
-        this.queryStartTimeNanos = queryStartTimeNanos;
         this.whereRunning = clusterAlias;
         // for the DataNodeHandler ComputeListener, clusterAlias and executionInfo will be null
         // for the top level ComputeListener in ComputeService both will be non-null
@@ -129,11 +124,15 @@ private ComputeListener(
             } else {
                 result = new ComputeResponse(collectedProfiles.isEmpty() ? List.of() : collectedProfiles.stream().toList());
                 if (coordinatingClusterIsSearchedInCCS()) {
-                    // mark local cluster as finished once the coordinator and all data nodes have finished processing
-                    executionInfo.swapCluster(
-                        RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
-                        (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL).build()
-                    );
+                    // if not already marked as SKIPPED, mark the local cluster as finished once the coordinator and all
+                    // data nodes have finished processing
+                    executionInfo.swapCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, (k, v) -> {
+                        if (v.getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) {
+                            return new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL).build();
+                        } else {
+                            return v;
+                        }
+                    });
                 }
             }
             delegate.onResponse(result);
@@ -196,8 +195,8 @@ ActionListener<Void> acquireAvoid() {
      *                            info to be gathered (namely, the DataNodeRequestHandler ComputeListener) should pass in null.
      */
     ActionListener<ComputeResponse> acquireCompute(@Nullable String computeClusterAlias) {
-        assert computeClusterAlias == null || (esqlExecutionInfo != null && queryStartTimeNanos > 0)
-            : "When clusterAlias is provided to acquireCompute, executionInfo must be non-null and queryStartTimeNanos must be positive";
+        assert computeClusterAlias == null || (esqlExecutionInfo != null && esqlExecutionInfo.getRelativeStartNanos() != null)
+            : "When clusterAlias is provided to acquireCompute, executionInfo and relativeStartTimeNanos must be non-null";
 
         return acquireAvoid().map(resp -> {
             responseHeaders.collect();
@@ -209,24 +208,17 @@ ActionListener<ComputeResponse> acquireCompute(@Nullable String computeClusterAl
                 return null;
             }
             if (isCCSListener(computeClusterAlias)) {
-                // this is the callback for the listener to the CCS compute
-                esqlExecutionInfo.swapCluster(
-                    computeClusterAlias,
-                    (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v)
-                        // for now ESQL doesn't return partial results, so set status to SUCCESSFUL
-                        .setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)
-                        .setTook(resp.getTook())
-                        .setTotalShards(resp.getTotalShards())
-                        .setSuccessfulShards(resp.getSuccessfulShards())
-                        .setSkippedShards(resp.getSkippedShards())
-                        .setFailedShards(resp.getFailedShards())
-                        .build()
-                );
+                // this is the callback for the listener on the primary coordinator that receives a remote ComputeResponse
+                updateExecutionInfoWithRemoteResponse(computeClusterAlias, resp);
+
             } else if (shouldRecordTookTime()) {
+                Long relativeStartNanos = esqlExecutionInfo.getRelativeStartNanos();
                 // handler for this cluster's data node and coordinator completion (runs on "local" and remote clusters)
-                TimeValue tookTime = new TimeValue(System.nanoTime() - queryStartTimeNanos, TimeUnit.NANOSECONDS);
+                assert relativeStartNanos != null : "queryStartTimeNanos not set properly";
+                TimeValue tookTime = new TimeValue(System.nanoTime() - relativeStartNanos, TimeUnit.NANOSECONDS);
                 esqlExecutionInfo.swapCluster(computeClusterAlias, (k, v) -> {
-                    if (v.getTook() == null || v.getTook().nanos() < tookTime.nanos()) {
+                    if (v.getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED
+                        && (v.getTook() == null || v.getTook().nanos() < tookTime.nanos())) {
                         return new EsqlExecutionInfo.Cluster.Builder(v).setTook(tookTime).build();
                     } else {
                         return v;
@@ -237,6 +229,40 @@ ActionListener<ComputeResponse> acquireCompute(@Nullable String computeClusterAl
         });
     }
 
+    private void updateExecutionInfoWithRemoteResponse(String computeClusterAlias, ComputeResponse resp) {
+        TimeValue tookOnCluster;
+        if (resp.getTook() != null) {
+            TimeValue remoteExecutionTime = resp.getTook();
+            TimeValue planningTookTime = esqlExecutionInfo.planningTookTime();
+            tookOnCluster = new TimeValue(planningTookTime.nanos() + remoteExecutionTime.nanos(), TimeUnit.NANOSECONDS);
+            esqlExecutionInfo.swapCluster(
+                computeClusterAlias,
+                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v)
+                    // for now ESQL doesn't return partial results, so set status to SUCCESSFUL
+                    .setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)
+                    .setTook(tookOnCluster)
+                    .setTotalShards(resp.getTotalShards())
+                    .setSuccessfulShards(resp.getSuccessfulShards())
+                    .setSkippedShards(resp.getSkippedShards())
+                    .setFailedShards(resp.getFailedShards())
+                    .build()
+            );
+        } else {
+            // if the cluster is an older version and does not send back took time, then calculate it here on the coordinator
+            // and leave shard info unset, so it is not shown in the CCS metadata section of the JSON response
+            long remoteTook = System.nanoTime() - esqlExecutionInfo.getRelativeStartNanos();
+            tookOnCluster = new TimeValue(remoteTook, TimeUnit.NANOSECONDS);
+            esqlExecutionInfo.swapCluster(
+                computeClusterAlias,
+                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v)
+                    // for now ESQL doesn't return partial results, so set status to SUCCESSFUL
+                    .setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)
+                    .setTook(tookOnCluster)
+                    .build()
+            );
+        }
+    }
+
     /**
      * Use this method when no execution metadata needs to be added to {@link EsqlExecutionInfo}
      */
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
index ce2a1d7a5f660..108e70d7d3a50 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -81,7 +80,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.xpack.esql.plugin.EsqlPlugin.ESQL_WORKER_THREAD_POOL_NAME;
@@ -173,19 +171,10 @@ public void execute(
                 null
             );
             String local = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
-            try (
-                var computeListener = ComputeListener.create(
-                    local,
-                    transportService,
-                    rootTask,
-                    execInfo,
-                    configuration.getQueryStartTimeNanos(),
-                    listener.map(r -> {
-                        updateExecutionInfoAfterCoordinatorOnlyQuery(configuration.getQueryStartTimeNanos(), execInfo);
-                        return new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo);
-                    })
-                )
-            ) {
+            try (var computeListener = ComputeListener.create(local, transportService, rootTask, execInfo, listener.map(r -> {
+                updateExecutionInfoAfterCoordinatorOnlyQuery(execInfo);
+                return new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo);
+            }))) {
                 runCompute(rootTask, computeContext, coordinatorPlan, computeListener.acquireCompute(local));
                 return;
             }
@@ -205,7 +194,6 @@ public void execute(
             queryPragmas.exchangeBufferSize(),
             transportService.getThreadPool().executor(ThreadPool.Names.SEARCH)
         );
-        long start = configuration.getQueryStartTimeNanos();
         String local = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
         /*
          * Grab the output attributes here, so we can pass them to
@@ -216,9 +204,8 @@ public void execute(
         try (
             Releasable ignored = exchangeSource.addEmptySink();
             // this is the top level ComputeListener called once at the end (e.g., once all clusters have finished for a CCS)
-            var computeListener = ComputeListener.create(local, transportService, rootTask, execInfo, start, listener.map(r -> {
-                long tookTimeNanos = System.nanoTime() - configuration.getQueryStartTimeNanos();
-                execInfo.overallTook(new TimeValue(tookTimeNanos, TimeUnit.NANOSECONDS));
+            var computeListener = ComputeListener.create(local, transportService, rootTask, execInfo, listener.map(r -> {
+                execInfo.markEndQuery();  // TODO: revisit this time recording model as part of INLINESTATS improvements
                 return new Result(outputAttributes, collectedPages, r.getProfiles(), execInfo);
             }))
         ) {
@@ -258,22 +245,24 @@ public void execute(
         }
     }
 
-    private static void updateExecutionInfoAfterCoordinatorOnlyQuery(long queryStartNanos, EsqlExecutionInfo execInfo) {
-        long tookTimeNanos = System.nanoTime() - queryStartNanos;
-        execInfo.overallTook(new TimeValue(tookTimeNanos, TimeUnit.NANOSECONDS));
+    // For queries like: FROM logs* | LIMIT 0 (including cross-cluster LIMIT 0 queries)
+    private static void updateExecutionInfoAfterCoordinatorOnlyQuery(EsqlExecutionInfo execInfo) {
+        execInfo.markEndQuery();  // TODO: revisit this time recording model as part of INLINESTATS improvements
         if (execInfo.isCrossClusterSearch()) {
+            assert execInfo.planningTookTime() != null : "Planning took time should be set on EsqlExecutionInfo but is null";
             for (String clusterAlias : execInfo.clusterAliases()) {
-                // The local cluster 'took' time gets updated as part of the acquireCompute(local) call in the coordinator, so
-                // here we only need to update status for remote clusters since there are no remote ComputeListeners in this case.
-                // This happens in cross cluster searches that use LIMIT 0, e.g, FROM logs*,remote*:logs* | LIMIT 0.
-                if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) {
-                    execInfo.swapCluster(clusterAlias, (k, v) -> {
-                        if (v.getStatus() == EsqlExecutionInfo.Cluster.Status.RUNNING) {
-                            return new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL).build();
-                        } else {
-                            return v;
-                        }
-                    });
+                // took time and shard counts for SKIPPED clusters were added at end of planning, so only update other cases here
+                if (execInfo.getCluster(clusterAlias).getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) {
+                    execInfo.swapCluster(
+                        clusterAlias,
+                        (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setTook(execInfo.overallTook())
+                            .setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)
+                            .setTotalShards(0)
+                            .setSuccessfulShards(0)
+                            .setSkippedShards(0)
+                            .setFailedShards(0)
+                            .build()
+                    );
                 }
             }
         }
@@ -834,11 +823,10 @@ public void messageReceived(ClusterComputeRequest request, TransportChannel chan
              * execution metadata for ES|QL processing local to this cluster. The execution info will be copied into the
              * ComputeResponse that is sent back to the primary coordinating cluster.
              */
-            EsqlExecutionInfo execInfo = new EsqlExecutionInfo();
+            EsqlExecutionInfo execInfo = new EsqlExecutionInfo(true);
             execInfo.swapCluster(clusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(clusterAlias, Arrays.toString(request.indices())));
             CancellableTask cancellable = (CancellableTask) task;
-            long start = request.configuration().getQueryStartTimeNanos();
-            try (var computeListener = ComputeListener.create(clusterAlias, transportService, cancellable, execInfo, start, listener)) {
+            try (var computeListener = ComputeListener.create(clusterAlias, transportService, cancellable, execInfo, listener)) {
                 runComputeOnRemoteCluster(
                     clusterAlias,
                     request.sessionId(),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java
index 4b6a38a3e8762..266f07d22eaf5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java
@@ -134,7 +134,6 @@ public class EsqlFeatures implements FeatureSpecification {
 
     /**
      * Blocks can be labelled with {@link org.elasticsearch.compute.data.Block.MvOrdering#SORTED_ASCENDING} for optimizations.
-     * C.f. {@link org.elasticsearch.TransportVersions#ESQL_MV_ORDERING_SORTED_ASCENDING}.
      */
     public static final NodeFeature MV_ORDERING_SORTED_ASCENDING = new NodeFeature("esql.mv_ordering_sorted_ascending");
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java
index 915efe9302a92..17329ca2e0054 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java
@@ -37,10 +37,15 @@ public class EsqlMediaTypeParser {
      * format. If there is a {@code format} parameter we use that. If there
      * isn't but there is a {@code Accept} header then we use that. If there
      * isn't then we use the {@code Content-Type} header which is required.
+     *
+     * Also validates certain parameter combinations and throws IllegalArgumentException if invalid
+     * combinations are detected.
      */
     public static MediaType getResponseMediaType(RestRequest request, EsqlQueryRequest esqlRequest) {
         var mediaType = request.hasParam(URL_PARAM_FORMAT) ? mediaTypeFromParams(request) : mediaTypeFromHeaders(request);
-        return validateColumnarRequest(esqlRequest.columnar(), mediaType, request);
+        validateColumnarRequest(esqlRequest.columnar(), mediaType);
+        validateIncludeCCSMetadata(esqlRequest.includeCCSMetadata(), mediaType);
+        return checkNonNullMediaType(mediaType, request);
     }
 
     private static MediaType mediaTypeFromHeaders(RestRequest request) {
@@ -53,7 +58,7 @@ private static MediaType mediaTypeFromParams(RestRequest request) {
         return MEDIA_TYPE_REGISTRY.queryParamToMediaType(request.param(URL_PARAM_FORMAT));
     }
 
-    private static MediaType validateColumnarRequest(boolean requestIsColumnar, MediaType fromMediaType, RestRequest request) {
+    private static void validateColumnarRequest(boolean requestIsColumnar, MediaType fromMediaType) {
         if (requestIsColumnar && fromMediaType instanceof TextFormat) {
             throw new IllegalArgumentException(
                 "Invalid use of [columnar] argument: cannot be used in combination with "
@@ -61,7 +66,16 @@ private static MediaType validateColumnarRequest(boolean requestIsColumnar, Medi
                     + " formats"
             );
         }
-        return checkNonNullMediaType(fromMediaType, request);
+    }
+
+    private static void validateIncludeCCSMetadata(boolean includeCCSMetadata, MediaType fromMediaType) {
+        if (includeCCSMetadata && fromMediaType instanceof TextFormat) {
+            throw new IllegalArgumentException(
+                "Invalid use of [include_ccs_metadata] argument: cannot be used in combination with "
+                    + Arrays.stream(TextFormat.values()).map(MediaType::queryParameter).toList()
+                    + " formats"
+            );
+        }
     }
 
     private static MediaType checkNonNullMediaType(MediaType mediaType, RestRequest request) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
index 17c795f2de28c..193930cdf711d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
@@ -167,7 +167,10 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<Es
             System.nanoTime()
         );
         String sessionId = sessionID(task);
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(clusterAlias -> remoteClusterService.isSkipUnavailable(clusterAlias));
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(
+            clusterAlias -> remoteClusterService.isSkipUnavailable(clusterAlias),
+            request.includeCCSMetadata()
+        );
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase = (physicalPlan, resultListener) -> computeService.execute(
             sessionId,
             (CancellableTask) task,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java
index 214c7b1053359..f6668db52b93b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java
@@ -69,14 +69,6 @@ public String toString(String field) {
     @Override
     public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) {
         return new ConstantScoreWeight(this, boost) {
-            @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
-                final ScorerSupplier scorerSupplier = scorerSupplier(context);
-                if (scorerSupplier == null) {
-                    return null;
-                }
-                return scorerSupplier.get(Long.MAX_VALUE);
-            }
 
             @Override
             public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
@@ -96,12 +88,12 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
                  * can't do that because we need the check the number of fields.
                  */
                 if (lfd instanceof LeafNumericFieldData n) {
-                    return scorerSupplier(context, n.getLongValues(), this, boost, scoreMode);
+                    return scorerSupplier(context, n.getLongValues(), boost, scoreMode);
                 }
                 if (lfd instanceof LeafOrdinalsFieldData o) {
-                    return scorerSupplier(context, o.getOrdinalsValues(), this, boost, scoreMode);
+                    return scorerSupplier(context, o.getOrdinalsValues(), boost, scoreMode);
                 }
-                return scorerSupplier(context, lfd.getBytesValues(), this, boost, scoreMode);
+                return scorerSupplier(context, lfd.getBytesValues(), boost, scoreMode);
             }
 
             @Override
@@ -113,7 +105,6 @@ public boolean isCacheable(LeafReaderContext ctx) {
             private ScorerSupplier scorerSupplier(
                 LeafReaderContext context,
                 SortedNumericDocValues sortedNumerics,
-                Weight weight,
                 float boost,
                 ScoreMode scoreMode
             ) throws IOException {
@@ -122,16 +113,9 @@ private ScorerSupplier scorerSupplier(
                     // check for dense field
                     final PointValues points = context.reader().getPointValues(fieldData.getFieldName());
                     if (points != null && points.getDocCount() == maxDoc) {
-                        return new DocIdSetIteratorScorerSupplier(weight, boost, scoreMode, DocIdSetIterator.all(maxDoc));
+                        return new DocIdSetIteratorScorerSupplier(boost, scoreMode, DocIdSetIterator.all(maxDoc));
                     } else {
-                        return new PredicateScorerSupplier(
-                            weight,
-                            boost,
-                            scoreMode,
-                            maxDoc,
-                            MULTI_VALUE_MATCH_COST,
-                            sortedNumerics::advanceExact
-                        );
+                        return new PredicateScorerSupplier(boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, sortedNumerics::advanceExact);
                     }
                 }
                 final CheckedIntPredicate predicate = doc -> {
@@ -144,13 +128,12 @@ private ScorerSupplier scorerSupplier(
                     }
                     return true;
                 };
-                return new PredicateScorerSupplier(weight, boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate);
+                return new PredicateScorerSupplier(boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate);
             }
 
             private ScorerSupplier scorerSupplier(
                 LeafReaderContext context,
                 SortedSetDocValues sortedSetDocValues,
-                Weight weight,
                 float boost,
                 ScoreMode scoreMode
             ) throws IOException {
@@ -159,10 +142,9 @@ private ScorerSupplier scorerSupplier(
                     // check for dense field
                     final Terms terms = context.reader().terms(fieldData.getFieldName());
                     if (terms != null && terms.getDocCount() == maxDoc) {
-                        return new DocIdSetIteratorScorerSupplier(weight, boost, scoreMode, DocIdSetIterator.all(maxDoc));
+                        return new DocIdSetIteratorScorerSupplier(boost, scoreMode, DocIdSetIterator.all(maxDoc));
                     } else {
                         return new PredicateScorerSupplier(
-                            weight,
                             boost,
                             scoreMode,
                             maxDoc,
@@ -181,20 +163,18 @@ private ScorerSupplier scorerSupplier(
                     }
                     return true;
                 };
-                return new PredicateScorerSupplier(weight, boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate);
+                return new PredicateScorerSupplier(boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate);
             }
 
             private ScorerSupplier scorerSupplier(
                 LeafReaderContext context,
                 SortedBinaryDocValues sortedBinaryDocValues,
-                Weight weight,
                 float boost,
                 ScoreMode scoreMode
             ) {
                 final int maxDoc = context.reader().maxDoc();
                 if (FieldData.unwrapSingleton(sortedBinaryDocValues) != null) {
                     return new PredicateScorerSupplier(
-                        weight,
                         boost,
                         scoreMode,
                         maxDoc,
@@ -212,7 +192,7 @@ private ScorerSupplier scorerSupplier(
                     }
                     return true;
                 };
-                return new PredicateScorerSupplier(weight, boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate);
+                return new PredicateScorerSupplier(boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate);
             }
         };
     }
@@ -266,13 +246,11 @@ public int hashCode() {
 
     private static class DocIdSetIteratorScorerSupplier extends ScorerSupplier {
 
-        private final Weight weight;
         private final float score;
         private final ScoreMode scoreMode;
         private final DocIdSetIterator docIdSetIterator;
 
-        private DocIdSetIteratorScorerSupplier(Weight weight, float score, ScoreMode scoreMode, DocIdSetIterator docIdSetIterator) {
-            this.weight = weight;
+        private DocIdSetIteratorScorerSupplier(float score, ScoreMode scoreMode, DocIdSetIterator docIdSetIterator) {
             this.score = score;
             this.scoreMode = scoreMode;
             this.docIdSetIterator = docIdSetIterator;
@@ -280,7 +258,7 @@ private DocIdSetIteratorScorerSupplier(Weight weight, float score, ScoreMode sco
 
         @Override
         public Scorer get(long leadCost) {
-            return new ConstantScoreScorer(weight, score, scoreMode, docIdSetIterator);
+            return new ConstantScoreScorer(score, scoreMode, docIdSetIterator);
         }
 
         @Override
@@ -290,23 +268,13 @@ public long cost() {
     }
 
     private static class PredicateScorerSupplier extends ScorerSupplier {
-
-        private final Weight weight;
         private final float score;
         private final ScoreMode scoreMode;
         private final int maxDoc;
         private final int matchCost;
         private final CheckedIntPredicate predicate;
 
-        private PredicateScorerSupplier(
-            Weight weight,
-            float score,
-            ScoreMode scoreMode,
-            int maxDoc,
-            int matchCost,
-            CheckedIntPredicate predicate
-        ) {
-            this.weight = weight;
+        private PredicateScorerSupplier(float score, ScoreMode scoreMode, int maxDoc, int matchCost, CheckedIntPredicate predicate) {
             this.score = score;
             this.scoreMode = scoreMode;
             this.maxDoc = maxDoc;
@@ -327,7 +295,7 @@ public float matchCost() {
                     return matchCost;
                 }
             };
-            return new ConstantScoreScorer(weight, score, scoreMode, iterator);
+            return new ConstantScoreScorer(score, scoreMode, iterator);
         }
 
         @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
index b904f07bcb9c3..4ec2746b24ee4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
@@ -96,7 +96,7 @@ public Configuration(BlockStreamInput in) throws IOException {
         } else {
             this.profile = false;
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REQUEST_TABLES)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.tables = in.readImmutableMap(i1 -> i1.readImmutableMap(i2 -> new Column((BlockStreamInput) i2)));
         } else {
             this.tables = Map.of();
@@ -124,7 +124,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
             out.writeBoolean(profile);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REQUEST_TABLES)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeMap(tables, (o1, columns) -> o1.writeMap(columns, StreamOutput::writeWriteable));
         }
         if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
index 96391c841856f..ccd167942340c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
@@ -72,7 +72,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
@@ -163,6 +162,7 @@ public void executeOptimizedPlan(
     ) {
         LogicalPlan firstPhase = Phased.extractFirstPhase(optimizedPlan);
         if (firstPhase == null) {
+            updateExecutionInfoAtEndOfPlanning(executionInfo);
             runPhase.accept(logicalPlanToPhysicalPlan(optimizedPlan, request), listener);
         } else {
             executePhased(new ArrayList<>(), optimizedPlan, request, executionInfo, firstPhase, runPhase, listener);
@@ -246,7 +246,6 @@ private <T> void preAnalyze(
                 if (indexResolution.isValid()) {
                     updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
                     updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.getUnavailableClusters());
-                    updateTookTimeForRemoteClusters(executionInfo);
                     Set<String> newClusters = enrichPolicyResolver.groupIndicesPerCluster(
                         indexResolution.get().concreteIndices().toArray(String[]::new)
                     ).keySet();
@@ -267,68 +266,6 @@ private <T> void preAnalyze(
         }));
     }
 
-    // visible for testing
-    static void updateExecutionInfoWithUnavailableClusters(EsqlExecutionInfo executionInfo, Set<String> unavailableClusters) {
-        for (String clusterAlias : unavailableClusters) {
-            executionInfo.swapCluster(
-                clusterAlias,
-                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED).build()
-            );
-            // TODO: follow-on PR will set SKIPPED status when skip_unavailable=true and throw an exception when skip_un=false
-        }
-    }
-
-    // visible for testing
-    static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionInfo executionInfo, IndexResolution indexResolution) {
-        Set<String> clustersWithResolvedIndices = new HashSet<>();
-        // determine missing clusters
-        for (String indexName : indexResolution.get().indexNameWithModes().keySet()) {
-            clustersWithResolvedIndices.add(RemoteClusterAware.parseClusterAlias(indexName));
-        }
-        Set<String> clustersRequested = executionInfo.clusterAliases();
-        Set<String> clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices);
-        clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters());
-        /*
-         * These are clusters in the original request that are not present in the field-caps response. They were
-         * specified with an index or indices that do not exist, so the search on that cluster is done.
-         * Mark it as SKIPPED with 0 shards searched and took=0.
-         */
-        for (String c : clustersWithNoMatchingIndices) {
-            executionInfo.swapCluster(
-                c,
-                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED)
-                    .setTook(new TimeValue(0))
-                    .setTotalShards(0)
-                    .setSuccessfulShards(0)
-                    .setSkippedShards(0)
-                    .setFailedShards(0)
-                    .build()
-            );
-        }
-    }
-
-    private void updateTookTimeForRemoteClusters(EsqlExecutionInfo executionInfo) {
-        if (executionInfo.isCrossClusterSearch()) {
-            for (String clusterAlias : executionInfo.clusterAliases()) {
-                if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) {
-                    executionInfo.swapCluster(clusterAlias, (k, v) -> {
-                        if (v.getTook() == null && v.getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) {
-                            // set took time in case we are finished with the remote cluster (e.g., FROM foo | LIMIT 0).
-                            // this will be overwritten later if ES|QL operations happen on the remote cluster (the typical scenario)
-                            TimeValue took = new TimeValue(
-                                System.nanoTime() - configuration.getQueryStartTimeNanos(),
-                                TimeUnit.NANOSECONDS
-                            );
-                            return new EsqlExecutionInfo.Cluster.Builder(v).setTook(took).build();
-                        } else {
-                            return v;
-                        }
-                    });
-                }
-            }
-        }
-    }
-
     private void preAnalyzeIndices(
         LogicalPlan parsed,
         EsqlExecutionInfo executionInfo,
@@ -508,4 +445,66 @@ public PhysicalPlan optimizedPhysicalPlan(LogicalPlan optimizedPlan) {
         LOGGER.debug("Optimized physical plan:\n{}", plan);
         return plan;
     }
+
+    // visible for testing
+    static void updateExecutionInfoWithUnavailableClusters(EsqlExecutionInfo executionInfo, Set<String> unavailableClusters) {
+        for (String clusterAlias : unavailableClusters) {
+            executionInfo.swapCluster(
+                clusterAlias,
+                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED).build()
+            );
+            // TODO: follow-on PR will set SKIPPED status when skip_unavailable=true and throw an exception when skip_un=false
+        }
+    }
+
+    // visible for testing
+    static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionInfo executionInfo, IndexResolution indexResolution) {
+        Set<String> clustersWithResolvedIndices = new HashSet<>();
+        // determine missing clusters
+        for (String indexName : indexResolution.get().indexNameWithModes().keySet()) {
+            clustersWithResolvedIndices.add(RemoteClusterAware.parseClusterAlias(indexName));
+        }
+        Set<String> clustersRequested = executionInfo.clusterAliases();
+        Set<String> clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices);
+        clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters());
+        /*
+         * These are clusters in the original request that are not present in the field-caps response. They were
+         * specified with an index or indices that do not exist, so the search on that cluster is done.
+         * Mark it as SKIPPED with 0 shards searched and took=0.
+         */
+        for (String c : clustersWithNoMatchingIndices) {
+            executionInfo.swapCluster(
+                c,
+                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED)
+                    .setTook(new TimeValue(0))
+                    .setTotalShards(0)
+                    .setSuccessfulShards(0)
+                    .setSkippedShards(0)
+                    .setFailedShards(0)
+                    .build()
+            );
+        }
+    }
+
+    // visible for testing
+    static void updateExecutionInfoAtEndOfPlanning(EsqlExecutionInfo execInfo) {
+        // TODO: this logic assumes a single phase execution model, so it may need to altered once INLINESTATS is made CCS compatible
+        if (execInfo.isCrossClusterSearch()) {
+            execInfo.markEndPlanning();
+            for (String clusterAlias : execInfo.clusterAliases()) {
+                EsqlExecutionInfo.Cluster cluster = execInfo.getCluster(clusterAlias);
+                if (cluster.getStatus() == EsqlExecutionInfo.Cluster.Status.SKIPPED) {
+                    execInfo.swapCluster(
+                        clusterAlias,
+                        (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setTook(execInfo.planningTookTime())
+                            .setTotalShards(0)
+                            .setSuccessfulShards(0)
+                            .setSkippedShards(0)
+                            .setFailedShards(0)
+                            .build()
+                    );
+                }
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java
index 6c5d9faf18ac4..092fecb3142db 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java
@@ -10,8 +10,11 @@
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -36,10 +39,17 @@ public String toString() {
     private final Map<QueryMetric, Map<OperationType, CounterMetric>> opsByTypeMetrics;
     // map that holds one counter per esql query "feature" (eval, sort, limit, where....)
     private final Map<FeatureMetric, CounterMetric> featuresMetrics;
+    private final Map<String, CounterMetric> functionMetrics;
     protected static String QPREFIX = "queries.";
     protected static String FPREFIX = "features.";
+    protected static String FUNC_PREFIX = "functions.";
 
-    public Metrics() {
+    private final EsqlFunctionRegistry functionRegistry;
+    private final Map<Class<?>, String> classToFunctionName;
+
+    public Metrics(EsqlFunctionRegistry functionRegistry) {
+        this.functionRegistry = functionRegistry.snapshotRegistry();
+        this.classToFunctionName = initClassToFunctionType();
         Map<QueryMetric, Map<OperationType, CounterMetric>> qMap = new LinkedHashMap<>();
         for (QueryMetric metric : QueryMetric.values()) {
             Map<OperationType, CounterMetric> metricsMap = Maps.newLinkedHashMapWithExpectedSize(OperationType.values().length);
@@ -56,6 +66,26 @@ public Metrics() {
             fMap.put(featureMetric, new CounterMetric());
         }
         featuresMetrics = Collections.unmodifiableMap(fMap);
+
+        functionMetrics = initFunctionMetrics();
+    }
+
+    private Map<String, CounterMetric> initFunctionMetrics() {
+        Map<String, CounterMetric> result = new LinkedHashMap<>();
+        for (var entry : classToFunctionName.entrySet()) {
+            result.put(entry.getValue(), new CounterMetric());
+        }
+        return Collections.unmodifiableMap(result);
+    }
+
+    private Map<Class<?>, String> initClassToFunctionType() {
+        Map<Class<?>, String> tmp = new HashMap<>();
+        for (FunctionDefinition func : functionRegistry.listFunctions()) {
+            if (tmp.containsKey(func.clazz()) == false) {
+                tmp.put(func.clazz(), func.name());
+            }
+        }
+        return Collections.unmodifiableMap(tmp);
     }
 
     /**
@@ -81,6 +111,13 @@ public void inc(FeatureMetric metric) {
         this.featuresMetrics.get(metric).inc();
     }
 
+    public void incFunctionMetric(Class<?> functionType) {
+        String functionName = classToFunctionName.get(functionType);
+        if (functionName != null) {
+            functionMetrics.get(functionName).inc();
+        }
+    }
+
     public Counters stats() {
         Counters counters = new Counters();
 
@@ -102,6 +139,11 @@ public Counters stats() {
             counters.inc(FPREFIX + entry.getKey().toString(), entry.getValue().count());
         }
 
+        // function metrics
+        for (Entry<String, CounterMetric> entry : functionMetrics.entrySet()) {
+            counters.inc(FUNC_PREFIX + entry.getKey(), entry.getValue().count());
+        }
+
         return counters;
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
index 3eef31e1cc406..3119fd4b52153 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
@@ -67,10 +67,7 @@
 import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
-import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext;
-import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.optimizer.TestLocalPhysicalPlanOptimizer;
-import org.elasticsearch.xpack.esql.optimizer.TestPhysicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -167,7 +164,6 @@ public class CsvTests extends ESTestCase {
     private final EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry();
     private final EsqlParser parser = new EsqlParser();
     private final Mapper mapper = new Mapper(functionRegistry);
-    private final PhysicalPlanOptimizer physicalPlanOptimizer = new TestPhysicalPlanOptimizer(new PhysicalOptimizerContext(configuration));
     private ThreadPool threadPool;
     private Executor executor;
 
@@ -252,12 +248,20 @@ public final void test() throws Throwable {
                 "can't use QSTR function in csv tests",
                 testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName())
             );
+            assumeFalse(
+                "can't use MATCH function in csv tests",
+                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_FUNCTION.capabilityName())
+            );
 
             if (Build.current().isSnapshot()) {
                 assertThat(
                     "Capability is not included in the enabled list capabilities on a snapshot build. Spelling mistake?",
                     testCase.requiredCapabilities,
-                    everyItem(in(EsqlCapabilities.CAPABILITIES))
+                    everyItem(in(EsqlCapabilities.capabilities(true)))
+                );
+                assumeTrue(
+                    "Capability not supported in this build",
+                    EsqlCapabilities.capabilities(false).containsAll(testCase.requiredCapabilities)
                 );
             } else {
                 for (EsqlCapabilities.Cap c : EsqlCapabilities.Cap.values()) {
@@ -433,7 +437,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception {
 
         session.executeOptimizedPlan(
             new EsqlQueryRequest(),
-            new EsqlExecutionInfo(),
+            new EsqlExecutionInfo(randomBoolean()),
             runPhase(bigArrays, physicalOperationProviders),
             session.optimizedPlan(analyzed),
             listener.delegateFailureAndWrap(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
index 0123186ced1aa..dcb83dadfcf96 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
@@ -100,29 +100,19 @@ public void testNamedParams() throws IOException {
         QueryBuilder filter = randomQueryBuilder();
 
         String paramsString = """
-            ,"params":[ {"n1" : {"value" : "f1", "kind" : "Identifier"}},
-             {"n2" : {"value" : "f1*", "Kind" : "identifier"}},
-             {"n3" : {"value" : "f.1*", "KIND" : "Pattern"}},
-             {"n4" : {"value" : "*", "kind" : "pattern"}},
-             {"n5" : {"value" : "esql", "kind" : "Value"}},
-             {"n_6" : {"value" : "null", "kind" : "identifier"}},
-             {"n7_" : {"value" : "f.1.1"}},
-             {"_n1" : "8.15.0"},
-             { "__n2" : 0.05},
-             {"__3" : -799810013},
-             {"__4n" : "127.0.0.1"},
-             {"_n5" : "esql"},
-             {"_n6" : null},
-             {"_n7" : false}] }""";
+            ,"params":[ {"n1" : "8.15.0"}, { "n2" : 0.05}, {"n3" : -799810013},
+             {"n4" : "127.0.0.1"}, {"n5" : "esql"}, {"n_6" : null}, {"n7_" : false},
+             {"_n1" : "8.15.0"}, { "__n2" : 0.05}, {"__3" : -799810013},
+             {"__4n" : "127.0.0.1"}, {"_n5" : "esql"}, {"_n6" : null}, {"_n7" : false}] }""";
 
         List<QueryParam> params = List.of(
-            paramAsIdentifier("n1", "f1"),
-            paramAsIdentifier("n2", "f1*"),
-            paramAsPattern("n3", "f.1*"),
-            paramAsPattern("n4", "*"),
+            paramAsConstant("n1", "8.15.0"),
+            paramAsConstant("n2", 0.05),
+            paramAsConstant("n3", -799810013),
+            paramAsConstant("n4", "127.0.0.1"),
             paramAsConstant("n5", "esql"),
-            paramAsIdentifier("n_6", "null"),
-            paramAsConstant("n7_", "f.1.1"),
+            paramAsConstant("n_6", null),
+            paramAsConstant("n7_", false),
             paramAsConstant("_n1", "8.15.0"),
             paramAsConstant("__n2", 0.05),
             paramAsConstant("__3", -799810013),
@@ -153,6 +143,56 @@ public void testNamedParams() throws IOException {
         }
     }
 
+    public void testNamedParamsForIdentifiersPatterns() throws IOException {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
+        String query = randomAlphaOfLengthBetween(1, 100);
+        boolean columnar = randomBoolean();
+        Locale locale = randomLocale(random());
+        QueryBuilder filter = randomQueryBuilder();
+
+        String paramsString = """
+            ,"params":[ {"n1" : {"identifier" : "f1"}},
+             {"n2" : {"Identifier" : "f1*"}},
+             {"n3" : {"pattern" : "f.1*"}},
+             {"n4" : {"Pattern" : "*"}},
+             {"n5" : {"Value" : "esql"}},
+             {"n_6" : {"identifier" : "null"}},
+             {"n7_" : {"value" : "f.1.1"}}] }""";
+
+        List<QueryParam> params = List.of(
+            paramAsIdentifier("n1", "f1"),
+            paramAsIdentifier("n2", "f1*"),
+            paramAsPattern("n3", "f.1*"),
+            paramAsPattern("n4", "*"),
+            paramAsConstant("n5", "esql"),
+            paramAsIdentifier("n_6", "null"),
+            paramAsConstant("n7_", "f.1.1")
+        );
+        String json = String.format(Locale.ROOT, """
+            {
+                "query": "%s",
+                "columnar": %s,
+                "locale": "%s",
+                "filter": %s
+                %s""", query, columnar, locale.toLanguageTag(), filter, paramsString);
+
+        EsqlQueryRequest request = parseEsqlQueryRequestSync(json);
+
+        assertEquals(query, request.query());
+        assertEquals(columnar, request.columnar());
+        assertEquals(locale.toLanguageTag(), request.locale().toLanguageTag());
+        assertEquals(locale, request.locale());
+        assertEquals(filter, request.filter());
+        assertEquals(params.size(), request.params().size());
+
+        for (int i = 0; i < request.params().size(); i++) {
+            assertEquals(params.get(i), request.params().get(i + 1));
+        }
+    }
+
     public void testInvalidParams() throws IOException {
         String query = randomAlphaOfLengthBetween(1, 100);
         boolean columnar = randomBoolean();
@@ -217,49 +257,70 @@ public void testInvalidParams() throws IOException {
             e2.getCause().getMessage(),
             containsString("Params cannot contain both named and unnamed parameters; got [{1:v1}, {1x:v1}] and [{1}, {2}]")
         );
+    }
+
+    public void testInvalidParamsForIdentifiersPatterns() throws IOException {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
+        String query = randomAlphaOfLengthBetween(1, 100);
+        boolean columnar = randomBoolean();
+        Locale locale = randomLocale(random());
+        QueryBuilder filter = randomQueryBuilder();
 
         // invalid named parameter for identifier and identifier pattern
-        String paramsString3 = """
-            "params":[ {"n1" : {"v" : "v1"}}, {"n2" : {"value" : "v2", "type" : "identifier"}},
-            {"n3" : {"value" : "v3", "kind" : "id" }}, {"n4" : {"value" : "v4", "kind" : true}},
-            {"n5" : {"value" : "v5", "kind" : ["identifier", "pattern"]}}, {"n6" : {"value" : "v6", "kind" : 0}},
-            {"n7" : {"value" : 1, "kind" : "Identifier"}}, {"n8" : {"value" : true, "kind" : "Pattern"}},
-            {"n9" : {"kind" : "identifier"}}, {"n10" : {"v" : "v10", "kind" : "identifier"}},
-            {"n11" : {"value" : "v11", "kind" : "pattern"}}, {"n12" : {"value" : ["x", "y"], "kind" : "identifier"}},
-            {"n13" : {"value" : "v13", "kind" : "identifier", "type" : "pattern"}}, {"n14" : {"v" : "v14", "kind" : "value"}}]""";
-        String json3 = String.format(Locale.ROOT, """
+        String paramsString1 = """
+            "params":[{"n1" : {"v" : "v1"}}, {"n2" : {"identifier" : "v2", "pattern" : "v2"}},
+            {"n3" : {"identifier" : "v3", "pattern" : "v3"}}, {"n4" : {"pattern" : "v4.1", "value" : "v4.2"}},
+            {"n5" : {"value" : {"a5" : "v5"}}},{"n6" : {"identifier" : {"a6.1" : "v6.1", "a6.2" : "v6.2"}}}, {"n7" : {}},
+            {"n8" : {"value" : ["x", "y"]}}, {"n9" : {"identifier" : ["x", "y"]}}, {"n10" : {"pattern" : ["x*", "y*"]}},
+            {"n11" : {"identifier" : 1}}, {"n12" : {"pattern" : true}}, {"n13" : {"identifier" : null}}, {"n14" : {"pattern" : "v14"}},
+            {"n15" : {"pattern" : "v15*"}, "n16" : {"identifier" : "v16"}}]""";
+        String json1 = String.format(Locale.ROOT, """
             {
                 %s
                 "query": "%s",
                 "columnar": %s,
                 "locale": "%s",
                 "filter": %s
-            }""", paramsString3, query, columnar, locale.toLanguageTag(), filter);
+            }""", paramsString1, query, columnar, locale.toLanguageTag(), filter);
 
-        Exception e3 = expectThrows(XContentParseException.class, () -> parseEsqlQueryRequestSync(json3));
+        Exception e1 = expectThrows(XContentParseException.class, () -> parseEsqlQueryRequestSync(json1));
         assertThat(
-            e3.getCause().getMessage(),
+            e1.getCause().getMessage(),
             containsString(
-                "Failed to parse params: [2:16] [v] is not a valid param attribute, a valid attribute is any of VALUE, KIND; "
-                    + "[2:39] [type] is not a valid param attribute, a valid attribute is any of VALUE, KIND; "
-                    + "[3:1] [id] is not a valid param kind, a valid kind is any of VALUE, IDENTIFIER, PATTERN; "
-                    + "[3:44] [true] is not a valid param kind, a valid kind is any of VALUE, IDENTIFIER, PATTERN; "
-                    + "[4:1] [[identifier, pattern]] is not a valid param kind, a valid kind is any of VALUE, IDENTIFIER, PATTERN; "
-                    + "[4:64] [0] is not a valid param kind, a valid kind is any of VALUE, IDENTIFIER, PATTERN; "
-                    + "[5:1] [1] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; "
-                    + "[5:48] [true] is not a valid value for PATTERN parameter, "
+                "[2:15] [v] is not a valid param attribute, a valid attribute is any of VALUE, IDENTIFIER, PATTERN; "
+                    + "[2:38] [n2] has multiple param attributes [identifier, pattern], "
+                    + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; "
+                    + "[2:38] [v2] is not a valid value for PATTERN parameter, "
+                    + "a valid value for PATTERN parameter is a string and contains *; "
+                    + "[3:1] [n3] has multiple param attributes [identifier, pattern], "
+                    + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; "
+                    + "[3:1] [v3] is not a valid value for PATTERN parameter, "
+                    + "a valid value for PATTERN parameter is a string and contains *; "
+                    + "[3:51] [n4] has multiple param attributes [pattern, value], "
+                    + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; "
+                    + "[3:51] [v4.1] is not a valid value for PATTERN parameter, "
+                    + "a valid value for PATTERN parameter is a string and contains *; "
+                    + "[4:1] n5={value={a5=v5}} is not supported as a parameter; "
+                    + "[4:36] [{a6.1=v6.1, a6.2=v6.2}] is not a valid value for IDENTIFIER parameter, "
+                    + "a valid value for IDENTIFIER parameter is a string; "
+                    + "[4:36] n6={identifier={a6.1=v6.1, a6.2=v6.2}} is not supported as a parameter; "
+                    + "[4:98] [n7] has no valid param attribute, only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; "
+                    + "[5:1] n8={value=[x, y]} is not supported as a parameter; "
+                    + "[5:34] [[x, y]] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; "
+                    + "[5:34] n9={identifier=[x, y]} is not supported as a parameter; "
+                    + "[5:72] [[x*, y*]] is not a valid value for PATTERN parameter, "
+                    + "a valid value for PATTERN parameter is a string and contains *; "
+                    + "[5:72] n10={pattern=[x*, y*]} is not supported as a parameter; "
+                    + "[6:1] [1] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; "
+                    + "[6:31] [true] is not a valid value for PATTERN parameter, "
                     + "a valid value for PATTERN parameter is a string and contains *; "
-                    + "[6:1] [null] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; "
-                    + "[6:35] [v] is not a valid param attribute, a valid attribute is any of VALUE, KIND; "
-                    + "[6:35] [n10={v=v10, kind=identifier}] does not have a value specified; "
-                    + "[6:35] [null] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; "
-                    + "[7:1] [v11] is not a valid value for PATTERN parameter, "
+                    + "[6:61] [null] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; "
+                    + "[6:94] [v14] is not a valid value for PATTERN parameter, "
                     + "a valid value for PATTERN parameter is a string and contains *; "
-                    + "[7:50] [[x, y]] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; "
-                    + "[7:50] n12={kind=identifier, value=[x, y]} is not supported as a parameter; "
-                    + "[8:1] [type] is not a valid param attribute, a valid attribute is any of VALUE, KIND; "
-                    + "[8:73] [v] is not a valid param attribute, a valid attribute is any of VALUE, KIND; "
-                    + "[8:73] [n14={v=v14, kind=value}] does not have a value specified"
+                    + "[7:1] Cannot parse more than one key:value pair as parameter, found [{n16:{identifier=v16}}, {n15:{pattern=v15*}}]"
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
index abf03d4fe06dd..27343bf7ce205 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
@@ -134,7 +134,7 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil
     }
 
     EsqlExecutionInfo createExecutionInfo() {
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
         executionInfo.overallTook(new TimeValue(5000));
         executionInfo.swapCluster(
             "",
@@ -193,7 +193,7 @@ private Page randomPage(List<ColumnInfoImpl> columns) {
                 case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(randomInt());
                 case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(randomDouble());
                 case KEYWORD -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(randomAlphaOfLength(10)));
-                case TEXT -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(randomAlphaOfLength(10000)));
+                case TEXT, SEMANTIC_TEXT -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(randomAlphaOfLength(10000)));
                 case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(
                     new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())))
                 );
@@ -426,9 +426,9 @@ static EsqlExecutionInfo parseClusters(XContentParser parser) throws IOException
                 }
             }
             if (clusterInfoMap.isEmpty()) {
-                return new EsqlExecutionInfo();
+                return new EsqlExecutionInfo(true);
             } else {
-                return new EsqlExecutionInfo(clusterInfoMap);
+                return new EsqlExecutionInfo(clusterInfoMap, true);
             }
         }
 
@@ -866,7 +866,7 @@ static Page valuesToPage(BlockFactory blockFactory, List<ColumnInfoImpl> columns
                     case LONG, COUNTER_LONG -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue());
                     case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue());
                     case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue());
-                    case KEYWORD, TEXT -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString()));
+                    case KEYWORD, TEXT, SEMANTIC_TEXT -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString()));
                     case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull();
                     case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToIP(value.toString()));
                     case DATETIME -> {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 5d75549893512..c18f55a651408 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.LoadMapping;
 import org.elasticsearch.xpack.esql.VerificationException;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
@@ -1219,7 +1220,7 @@ public void testAggsOverGroupingKey() throws Exception {
         assertThat(output, hasSize(2));
         var aggs = agg.aggregates();
         var min = as(Alias.unwrap(aggs.get(0)), Min.class);
-        assertThat(min.arguments(), hasSize(1));
+        assertThat(min.arguments(), hasSize(2));    // field + filter
         var group = Alias.unwrap(agg.groupings().get(0));
         assertEquals(min.arguments().get(0), group);
     }
@@ -1241,7 +1242,7 @@ public void testAggsOverGroupingKeyWithAlias() throws Exception {
         assertThat(output, hasSize(2));
         var aggs = agg.aggregates();
         var min = as(Alias.unwrap(aggs.get(0)), Min.class);
-        assertThat(min.arguments(), hasSize(1));
+        assertThat(min.arguments(), hasSize(2));    // field + filter
         assertEquals(Expressions.attribute(min.arguments().get(0)), Expressions.attribute(agg.groupings().get(0)));
     }
 
@@ -1638,7 +1639,7 @@ public void testCounterTypes() {
         var attributes = limit.output().stream().collect(Collectors.toMap(NamedExpression::name, a -> a));
         assertThat(
             attributes.keySet(),
-            equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in"))
+            equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in", "network.message_out"))
         );
         assertThat(attributes.get("network.connections").dataType(), equalTo(DataType.LONG));
         assertThat(attributes.get("network.bytes_in").dataType(), equalTo(DataType.COUNTER_LONG));
@@ -2084,6 +2085,10 @@ public void testCoalesceWithMixedNumericTypes() {
     }
 
     public void testNamedParamsForIdentifiers() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         assertProjectionWithMapping(
             """
                 from test
@@ -2174,6 +2179,10 @@ public void testNamedParamsForIdentifiers() {
     }
 
     public void testInvalidNamedParamsForIdentifiers() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         // missing field
         assertError(
             """
@@ -2243,6 +2252,10 @@ public void testInvalidNamedParamsForIdentifiers() {
     }
 
     public void testNamedParamsForIdentifierPatterns() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         assertProjectionWithMapping(
             """
                 from test
@@ -2273,6 +2286,10 @@ public void testNamedParamsForIdentifierPatterns() {
     }
 
     public void testInvalidNamedParamsForIdentifierPatterns() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         // missing pattern
         assertError(
             """
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
index 83b3e88c57964..63f7629f3c720 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.analysis;
 
 import org.elasticsearch.Build;
+import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
@@ -25,12 +26,16 @@
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsConstant;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping;
+import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -355,6 +360,40 @@ public void testAggsInsideGrouping() {
         );
     }
 
+    public void testAggFilterOnNonAggregates() {
+        assertEquals(
+            "1:36: WHERE clause allowed only for aggregate functions, none found in [emp_no + 1 where languages > 1]",
+            error("from test | stats emp_no + 1 where languages > 1 by emp_no")
+        );
+        assertEquals(
+            "1:53: WHERE clause allowed only for aggregate functions, none found in [abs(emp_no + languages) % 2 WHERE languages > 1]",
+            error("from test | stats abs(emp_no + languages) % 2 WHERE languages > 1 by emp_no, languages")
+        );
+    }
+
+    public void testAggFilterOnBucketingOrAggFunctions() {
+        // query passes when the bucket function is part of the BY clause
+        query("from test | stats max(languages) WHERE bucket(salary, 10) > 1 by bucket(salary, 10)");
+
+        // but fails if it's different
+        assertEquals(
+            "1:40: can only use grouping function [bucket(salary, 10)] part of the BY clause",
+            error("from test | stats max(languages) WHERE bucket(salary, 10) > 1 by emp_no")
+        );
+
+        assertEquals(
+            "1:40: cannot use aggregate function [max(salary)] in aggregate WHERE clause [max(languages) WHERE max(salary) > 1]",
+            error("from test | stats max(languages) WHERE max(salary) > 1 by emp_no")
+        );
+
+        assertEquals(
+            "1:40: cannot use aggregate function [max(salary)] in aggregate WHERE clause [max(languages) WHERE max(salary) + 2 > 1]",
+            error("from test | stats max(languages) WHERE max(salary) + 2 > 1 by emp_no")
+        );
+
+        assertEquals("1:60: Unknown column [m]", error("from test | stats m = max(languages), min(languages) WHERE m + 2 > 1 by emp_no"));
+    }
+
     public void testGroupingInsideAggsAsAgg() {
         assertEquals(
             "1:18: can only use grouping function [bucket(emp_no, 5.)] part of the BY clause",
@@ -908,6 +947,25 @@ public void testSpatialSort() {
         assertEquals("1:42: cannot sort on cartesian_shape", error("FROM countries_bbox_web | LIMIT 5 | sort shape", countriesBboxWeb));
     }
 
+    public void testSourceSorting() {
+        assertEquals("1:35: cannot sort on _source", error("from test metadata _source | sort _source"));
+    }
+
+    public void testCountersSorting() {
+        Map<DataType, String> counterDataTypes = Map.of(
+            COUNTER_DOUBLE,
+            "network.message_in",
+            COUNTER_INTEGER,
+            "network.message_out",
+            COUNTER_LONG,
+            "network.bytes_out"
+        );
+        for (DataType counterDT : counterDataTypes.keySet()) {
+            var fieldName = counterDataTypes.get(counterDT);
+            assertEquals("1:18: cannot sort on " + counterDT.name().toLowerCase(Locale.ROOT), error("from test | sort " + fieldName, tsdb));
+        }
+    }
+
     public void testInlineImpossibleConvert() {
         assertEquals("1:5: argument of [false::ip] must be [ip or string], found value [false] type [boolean]", error("ROW false::ip"));
     }
@@ -1086,9 +1144,14 @@ public void testMatchFilter() throws Exception {
         );
     }
 
-    public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {
-        assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+    public void testMatchFunctionNotAllowedAfterCommands() throws Exception {
+        assertEquals(
+            "1:24: [MATCH] function cannot be used after LIMIT",
+            error("from test | limit 10 | where match(first_name, \"Anna\")")
+        );
+    }
 
+    public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {
         // Source commands
         assertEquals("1:13: [QSTR] function cannot be used after SHOW", error("show info | where qstr(\"8.16.0\")"));
         assertEquals("1:17: [QSTR] function cannot be used after ROW", error("row a= \"Anna\" | where qstr(\"Anna\")"));
@@ -1146,26 +1209,145 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {
         );
     }
 
-    public void testQueryStringFunctionsOnlyAllowedInWhere() throws Exception {
-        assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+    public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception {
+        assertEquals("1:9: [QSTR] function is only supported in WHERE commands", error("row a = qstr(\"Anna\")"));
+        checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")");
+    }
+
+    public void testMatchFunctionOnlyAllowedInWhere() throws Exception {
+        checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")");
+    }
 
-        assertEquals("1:22: [QSTR] function is only supported in WHERE commands", error("from test | eval y = qstr(\"Anna\")"));
-        assertEquals("1:18: [QSTR] function is only supported in WHERE commands", error("from test | sort qstr(\"Connection\") asc"));
-        assertEquals("1:5: [QSTR] function is only supported in WHERE commands", error("row qstr(\"Connection\")"));
+    private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation) throws Exception {
+        assertEquals(
+            "1:22: [" + functionName + "] function is only supported in WHERE commands",
+            error("from test | eval y = " + functionInvocation)
+        );
+        assertEquals(
+            "1:18: [" + functionName + "] function is only supported in WHERE commands",
+            error("from test | sort " + functionInvocation + " asc")
+        );
         assertEquals(
-            "1:23: [QSTR] function is only supported in WHERE commands",
-            error("from test | STATS c = qstr(\"foo\") BY languages")
+            "1:23: [" + functionName + "] function is only supported in WHERE commands",
+            error("from test | STATS c = " + functionInvocation + " BY first_name")
         );
     }
 
     public void testQueryStringFunctionArgNotNullOrConstant() throws Exception {
-        assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+        assertEquals(
+            "1:19: argument of [qstr(first_name)] must be a constant, received [first_name]",
+            error("from test | where qstr(first_name)")
+        );
+        assertEquals("1:19: argument of [qstr(null)] cannot be null, received [null]", error("from test | where qstr(null)"));
+        // Other value types are tested in QueryStringFunctionTests
+    }
 
-        assertEquals("1:19: argument of [QSTR] must be a constant, received [first_name]", error("from test | where qstr(first_name)"));
-        assertEquals("1:19: argument of [QSTR] cannot be null, received [null]", error("from test | where qstr(null)"));
+    public void testQueryStringWithDisjunctions() {
+        checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")");
+    }
+
+    public void testMatchWithDisjunctions() {
+        checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")");
+    }
+
+    private void checkWithDisjunctions(String functionName, String functionInvocation) {
+        assertEquals(
+            LoggerMessageFormat.format(
+                null,
+                "1:19: Invalid condition [{} or length(first_name) > 12]. " + "Function {} can't be used as part of an or condition",
+                functionInvocation,
+                functionName
+            ),
+            error("from test | where " + functionInvocation + " or length(first_name) > 12")
+        );
+        assertEquals(
+            LoggerMessageFormat.format(
+                null,
+                "1:19: Invalid condition [({} and first_name is not null) or (length(first_name) > 12 and first_name is null)]. "
+                    + "Function {} can't be used as part of an or condition",
+                functionInvocation,
+                functionName
+            ),
+            error(
+                "from test | where ("
+                    + functionInvocation
+                    + " and first_name is not null) or (length(first_name) > 12 and first_name is null)"
+            )
+        );
+        assertEquals(
+            LoggerMessageFormat.format(
+                null,
+                "1:19: Invalid condition [({} and first_name is not null) or first_name is null]. "
+                    + "Function {} can't be used as part of an or condition",
+                functionInvocation,
+                functionName
+            ),
+            error("from test | where (" + functionInvocation + " and first_name is not null) or first_name is null")
+        );
+    }
+
+    public void testQueryStringFunctionWithNonBooleanFunctions() {
+        checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")");
+    }
+
+    public void testMatchFunctionWithNonBooleanFunctions() {
+        checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")");
+    }
+
+    private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, String functionInvocation) {
+        assertEquals(
+            "1:19: Invalid condition [" + functionInvocation + " is not null]. Function " + functionName + " can't be used with ISNOTNULL",
+            error("from test | where " + functionInvocation + " is not null")
+        );
+        assertEquals(
+            "1:19: Invalid condition [" + functionInvocation + " is null]. Function " + functionName + " can't be used with ISNULL",
+            error("from test | where " + functionInvocation + " is null")
+        );
+        assertEquals(
+            "1:19: Invalid condition ["
+                + functionInvocation
+                + " in (\"hello\", \"world\")]. Function "
+                + functionName
+                + " can't be used with IN",
+            error("from test | where " + functionInvocation + " in (\"hello\", \"world\")")
+        );
+    }
+
+    public void testMatchFunctionArgNotConstant() throws Exception {
+        assertEquals(
+            "1:19: second argument of [match(first_name, first_name)] must be a constant, received [first_name]",
+            error("from test | where match(first_name, first_name)")
+        );
+        assertEquals(
+            "1:59: second argument of [match(first_name, query)] must be a constant, received [query]",
+            error("from test | eval query = concat(\"first\", \" name\") | where match(first_name, query)")
+        );
         // Other value types are tested in QueryStringFunctionTests
     }
 
+    // These should pass eventually once we lift some restrictions on match function
+    public void testMatchFunctionCurrentlyUnsupportedBehaviour() throws Exception {
+        assertEquals(
+            "1:68: Unknown column [first_name]",
+            error("from test | stats max_salary = max(salary) by emp_no | where match(first_name, \"Anna\")")
+        );
+    }
+
+    public void testMatchFunctionNullArgs() throws Exception {
+        assertEquals(
+            "1:19: first argument of [match(null, \"query\")] cannot be null, received [null]",
+            error("from test | where match(null, \"query\")")
+        );
+        assertEquals(
+            "1:19: second argument of [match(first_name, null)] cannot be null, received [null]",
+            error("from test | where match(first_name, null)")
+        );
+    }
+
+    public void testMatchFunctionTargetsExistingField() throws Exception {
+        assertEquals("1:39: Unknown column [first_name]", error("from test | keep emp_no | where match(first_name, \"Anna\")"));
+    }
+
     public void testCoalesceWithMixedNumericTypes() {
         assertEquals(
             "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]",
@@ -1359,6 +1541,10 @@ public void testToDatePeriodToTimeDurationWithInvalidType() {
         );
     }
 
+    private void query(String query) {
+        defaultAnalyzer.analyze(parser.createStatement(query));
+    }
+
     private String error(String query) {
         return error(query, defaultAnalyzer);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java
index ebad8e6e13b8c..39170f1a305df 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java
@@ -446,9 +446,9 @@ EnrichResolution resolvePolicies(Collection<String> clusters, Collection<Unresol
         }
 
         @Override
-        protected Transport.Connection getRemoteConnection(String remoteCluster) {
+        protected void getRemoteConnection(String remoteCluster, ActionListener<Transport.Connection> listener) {
             assertThat("Must only called on the local cluster", cluster, equalTo(LOCAL_CLUSTER_GROUP_KEY));
-            return transports.get("").getConnection(transports.get(remoteCluster).getLocalDiscoNode());
+            listener.onResponse(transports.get("").getConnection(transports.get(remoteCluster).getLocalNode()));
         }
 
         static ClusterService mockClusterService(Map<String, EnrichPolicy> policies) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java
index 107c2af11c4f1..04da5d406fbb9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java
@@ -108,7 +108,7 @@ public void testQueries() throws Exception {
         QueryList queryList = QueryList.termQueryList(uidField, mock(SearchExecutionContext.class), inputTerms, KEYWORD);
         assertThat(queryList.getPositionCount(), equalTo(6));
         assertThat(queryList.getQuery(0), equalTo(new TermQuery(new Term("uid", new BytesRef("b2")))));
-        assertThat(queryList.getQuery(1), equalTo(new TermInSetQuery("uid", new BytesRef("c1"), new BytesRef("a2"))));
+        assertThat(queryList.getQuery(1), equalTo(new TermInSetQuery("uid", List.of(new BytesRef("c1"), new BytesRef("a2")))));
         assertThat(queryList.getQuery(2), equalTo(new TermQuery(new Term("uid", new BytesRef("z2")))));
         assertNull(queryList.getQuery(3));
         assertThat(queryList.getQuery(4), equalTo(new TermQuery(new Term("uid", new BytesRef("a3")))));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
index 1c917a961a343..db5d8e03458ea 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
@@ -163,15 +163,7 @@ private void aggregateSingleMode(Expression expression) {
             result = extractResultFromAggregator(aggregator, PlannerUtils.toElementType(testCase.expectedType()));
         }
 
-        assertThat(result, not(equalTo(Double.NaN)));
-        assert testCase.getMatcher().matches(Double.POSITIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.POSITIVE_INFINITY)));
-        assert testCase.getMatcher().matches(Double.NEGATIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.NEGATIVE_INFINITY)));
-        assertThat(result, testCase.getMatcher());
-        if (testCase.getExpectedWarnings() != null) {
-            assertWarnings(testCase.getExpectedWarnings());
-        }
+        assertTestCaseResultAndWarnings(result);
     }
 
     private void aggregateGroupingSingleMode(Expression expression) {
@@ -263,15 +255,7 @@ private void aggregateWithIntermediates(Expression expression) {
             result = extractResultFromAggregator(aggregator, PlannerUtils.toElementType(testCase.expectedType()));
         }
 
-        assertThat(result, not(equalTo(Double.NaN)));
-        assert testCase.getMatcher().matches(Double.POSITIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.POSITIVE_INFINITY)));
-        assert testCase.getMatcher().matches(Double.NEGATIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.NEGATIVE_INFINITY)));
-        assertThat(result, testCase.getMatcher());
-        if (testCase.getExpectedWarnings() != null) {
-            assertWarnings(testCase.getExpectedWarnings());
-        }
+        assertTestCaseResultAndWarnings(result);
     }
 
     private void evaluate(Expression evaluableExpression) {
@@ -288,15 +272,7 @@ private void evaluate(Expression evaluableExpression) {
         if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) {
             result = NumericUtils.unsignedLongAsBigInteger((Long) result);
         }
-        assertThat(result, not(equalTo(Double.NaN)));
-        assert testCase.getMatcher().matches(Double.POSITIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.POSITIVE_INFINITY)));
-        assert testCase.getMatcher().matches(Double.NEGATIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.NEGATIVE_INFINITY)));
-        assertThat(result, testCase.getMatcher());
-        if (testCase.getExpectedWarnings() != null) {
-            assertWarnings(testCase.getExpectedWarnings());
-        }
+        assertTestCaseResultAndWarnings(result);
     }
 
     private void resolveExpression(Expression expression, Consumer<Expression> onAggregator, Consumer<Expression> onEvaluableExpression) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
index ca9950a4bfe77..c05f8e0990b3c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
@@ -99,8 +99,10 @@
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
@@ -722,17 +724,19 @@ public static void testFunctionInfo() {
             for (int i = 0; i < args.size() && i < types.size(); i++) {
                 typesFromSignature.get(i).add(types.get(i).esNameIfPossible());
             }
-            returnFromSignature.add(entry.getValue().esNameIfPossible());
+            if (DataType.UNDER_CONSTRUCTION.containsKey(entry.getValue()) == false) {
+                returnFromSignature.add(entry.getValue().esNameIfPossible());
+            }
         }
 
         for (int i = 0; i < args.size(); i++) {
             EsqlFunctionRegistry.ArgSignature arg = args.get(i);
             Set<String> annotationTypes = Arrays.stream(arg.type())
-                .filter(DataType.UNDER_CONSTRUCTION::containsKey)
+                .filter(t -> DataType.UNDER_CONSTRUCTION.containsKey(DataType.fromNameOrAlias(t)) == false)
                 .collect(Collectors.toCollection(TreeSet::new));
             Set<String> signatureTypes = typesFromSignature.get(i)
                 .stream()
-                .filter(DataType.UNDER_CONSTRUCTION::containsKey)
+                .filter(t -> DataType.UNDER_CONSTRUCTION.containsKey(DataType.fromNameOrAlias(t)) == false)
                 .collect(Collectors.toCollection(TreeSet::new));
             if (signatureTypes.isEmpty()) {
                 log.info("{}: skipping", arg.name());
@@ -746,10 +750,40 @@ public static void testFunctionInfo() {
             );
         }
 
-        Set<String> returnTypes = Arrays.stream(description.returnType()).collect(Collectors.toCollection(TreeSet::new));
+        Set<String> returnTypes = Arrays.stream(description.returnType())
+            .filter(t -> DataType.UNDER_CONSTRUCTION.containsKey(DataType.fromNameOrAlias(t)) == false)
+            .collect(Collectors.toCollection(TreeSet::new));
         assertEquals(returnFromSignature, returnTypes);
     }
 
+    /**
+     * Asserts the result of a test case matches the expected result and warnings.
+     * <p>
+     * The {@code result} parameter should be an object as returned by {@link #toJavaObjectUnsignedLongAware}.
+     * </p>
+     */
+    @SuppressWarnings("unchecked")
+    protected final void assertTestCaseResultAndWarnings(Object result) {
+        if (result instanceof Iterable<?>) {
+            var collectionResult = (Iterable<Object>) result;
+            assertThat(collectionResult, not(hasItem(Double.NaN)));
+            assertThat(collectionResult, not(hasItem(Double.POSITIVE_INFINITY)));
+            assertThat(collectionResult, not(hasItem(Double.NEGATIVE_INFINITY)));
+        }
+
+        assert testCase.getMatcher().matches(Double.NaN) == false;
+        assertThat(result, not(equalTo(Double.NaN)));
+        assert testCase.getMatcher().matches(Double.POSITIVE_INFINITY) == false;
+        assertThat(result, not(equalTo(Double.POSITIVE_INFINITY)));
+        assert testCase.getMatcher().matches(Double.NEGATIVE_INFINITY) == false;
+        assertThat(result, not(equalTo(Double.NEGATIVE_INFINITY)));
+        assertThat(result, testCase.getMatcher());
+
+        if (testCase.getExpectedWarnings() != null) {
+            assertWarnings(testCase.getExpectedWarnings());
+        }
+    }
+
     protected final void assertTypeResolutionFailure(Expression expression) {
         assertTrue("expected unresolved", expression.typeResolved().unresolved());
         assertThat(expression.typeResolved().message(), equalTo(testCase.getExpectedTypeError()));
@@ -1283,13 +1317,6 @@ public void allMemoryReleased() {
         }
     }
 
-    /**
-     * All string types (keyword, text, match_only_text, etc).
-     */
-    protected static DataType[] strings() {
-        return DataType.types().stream().filter(DataType::isString).toArray(DataType[]::new);
-    }
-
     /**
      * Validate that we know the types for all the test cases already created
      * @param suppliers - list of suppliers before adding in the illegal type combinations
@@ -1316,10 +1343,9 @@ private static boolean isAggregation() {
      */
     private static boolean shouldHideSignature(List<DataType> argTypes, DataType returnType) {
         for (DataType dt : DataType.UNDER_CONSTRUCTION.keySet()) {
-            if (returnType == dt) {
+            if (returnType == dt || argTypes.contains(dt)) {
                 return true;
             }
-            return argTypes.contains(dt);
         }
         return false;
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
index 85db73901352b..65e8a53fc05c5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
@@ -41,7 +41,6 @@
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
@@ -127,15 +126,7 @@ public final void testEvaluate() {
                 result = toJavaObjectUnsignedLongAware(block, 0);
             }
         }
-        assertThat(result, not(equalTo(Double.NaN)));
-        assert testCase.getMatcher().matches(Double.POSITIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.POSITIVE_INFINITY)));
-        assert testCase.getMatcher().matches(Double.NEGATIVE_INFINITY) == false;
-        assertThat(result, not(equalTo(Double.NEGATIVE_INFINITY)));
-        assertThat(result, testCase.getMatcher());
-        if (testCase.getExpectedWarnings() != null) {
-            assertWarnings(testCase.getExpectedWarnings());
-        }
+        assertTestCaseResultAndWarnings(result);
     }
 
     /**
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
index e8f0333791844..6b2040f58f84c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
@@ -20,7 +20,7 @@
 public class FieldAttributeTests extends AbstractAttributeTestCase<FieldAttribute> {
     public static FieldAttribute createFieldAttribute(int maxDepth, boolean onlyRepresentable) {
         Source source = Source.EMPTY;
-        FieldAttribute parent = maxDepth == 0 || randomBoolean() ? null : createFieldAttribute(maxDepth - 1, onlyRepresentable);
+        String parentName = maxDepth == 0 || randomBoolean() ? null : randomAlphaOfLength(3);
         String name = randomAlphaOfLength(5);
         DataType type = onlyRepresentable
             ? randomValueOtherThanMany(t -> false == DataType.isRepresentable(t), () -> randomFrom(DataType.types()))
@@ -28,7 +28,7 @@ public static FieldAttribute createFieldAttribute(int maxDepth, boolean onlyRepr
         EsField field = AbstractEsFieldTypeTests.randomAnyEsField(maxDepth);
         Nullability nullability = randomFrom(Nullability.values());
         boolean synthetic = randomBoolean();
-        return newFieldAttributeWithType(source, parent, name, type, field, nullability, new NameId(), synthetic);
+        return newFieldAttributeWithType(source, parentName, name, type, field, nullability, new NameId(), synthetic);
     }
 
     @Override
@@ -39,20 +39,20 @@ protected FieldAttribute create() {
     @Override
     protected FieldAttribute mutate(FieldAttribute instance) {
         Source source = instance.source();
-        FieldAttribute parent = instance.parent();
+        String parentName = instance.parentName();
         String name = instance.name();
         DataType type = instance.dataType();
         EsField field = instance.field();
         Nullability nullability = instance.nullable();
         boolean synthetic = instance.synthetic();
         switch (between(0, 5)) {
-            case 0 -> parent = randomValueOtherThan(parent, () -> randomBoolean() ? null : createFieldAttribute(2, false));
+            case 0 -> parentName = randomValueOtherThan(parentName, () -> randomBoolean() ? null : randomAlphaOfLength(2));
             case 1 -> name = randomAlphaOfLength(name.length() + 1);
             case 2 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types()));
             case 3 -> field = randomValueOtherThan(field, () -> AbstractEsFieldTypeTests.randomAnyEsField(3));
             case 4 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
             case 5 -> synthetic = false == synthetic;
         }
-        return newFieldAttributeWithType(source, parent, name, type, field, nullability, new NameId(), synthetic);
+        return newFieldAttributeWithType(source, parentName, name, type, field, nullability, new NameId(), synthetic);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
index e740533462746..7fe67707a7976 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
@@ -19,6 +19,7 @@
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
 import static org.elasticsearch.test.ESTestCase.randomList;
@@ -37,56 +38,36 @@ public static List<TypedDataSupplier> intCases(int minRows, int maxRows, int min
         List<TypedDataSupplier> cases = new ArrayList<>();
 
         if (0 <= max && 0 >= min && includeZero) {
-            cases.add(new TypedDataSupplier("<0 ints>", () -> randomList(minRows, maxRows, () -> 0), DataType.INTEGER, false, true));
+            addSuppliers(cases, minRows, maxRows, "0 int", DataType.INTEGER, () -> 0);
         }
 
         if (max != 0) {
-            cases.add(
-                new TypedDataSupplier("<" + max + " ints>", () -> randomList(minRows, maxRows, () -> max), DataType.INTEGER, false, true)
-            );
+            addSuppliers(cases, minRows, maxRows, max + " int", DataType.INTEGER, () -> max);
         }
 
         if (min != 0 && min != max) {
-            cases.add(
-                new TypedDataSupplier("<" + min + " ints>", () -> randomList(minRows, maxRows, () -> min), DataType.INTEGER, false, true)
-            );
+            addSuppliers(cases, minRows, maxRows, min + " int", DataType.INTEGER, () -> min);
         }
 
         int lower = Math.max(min, 1);
         int upper = Math.min(max, Integer.MAX_VALUE);
         if (lower < upper) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<positive ints>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomIntBetween(lower, upper)),
-                    DataType.INTEGER,
-                    false,
-                    true
-                )
-            );
+            addSuppliers(cases, minRows, maxRows, "positive int", DataType.INTEGER, () -> ESTestCase.randomIntBetween(lower, upper));
         }
 
         int lower1 = Math.max(min, Integer.MIN_VALUE);
         int upper1 = Math.min(max, -1);
         if (lower1 < upper1) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<negative ints>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomIntBetween(lower1, upper1)),
-                    DataType.INTEGER,
-                    false,
-                    true
-                )
-            );
+            addSuppliers(cases, minRows, maxRows, "negative int", DataType.INTEGER, () -> ESTestCase.randomIntBetween(lower1, upper1));
         }
 
         if (min < 0 && max > 0) {
-            cases.add(new TypedDataSupplier("<random ints>", () -> randomList(minRows, maxRows, () -> {
+            addSuppliers(cases, minRows, maxRows, "random int", DataType.INTEGER, () -> {
                 if (includeZero) {
                     return ESTestCase.randomIntBetween(min, max);
                 }
                 return randomBoolean() ? ESTestCase.randomIntBetween(min, -1) : ESTestCase.randomIntBetween(1, max);
-            }), DataType.INTEGER, false, true));
+            });
         }
 
         return cases;
@@ -96,56 +77,36 @@ public static List<TypedDataSupplier> longCases(int minRows, int maxRows, long m
         List<TypedDataSupplier> cases = new ArrayList<>();
 
         if (0 <= max && 0 >= min && includeZero) {
-            cases.add(new TypedDataSupplier("<0 longs>", () -> randomList(minRows, maxRows, () -> 0L), DataType.LONG, false, true));
+            addSuppliers(cases, minRows, maxRows, "0 long", DataType.LONG, () -> 0L);
         }
 
         if (max != 0) {
-            cases.add(
-                new TypedDataSupplier("<" + max + " longs>", () -> randomList(minRows, maxRows, () -> max), DataType.LONG, false, true)
-            );
+            addSuppliers(cases, minRows, maxRows, max + " long", DataType.LONG, () -> max);
         }
 
         if (min != 0 && min != max) {
-            cases.add(
-                new TypedDataSupplier("<" + min + " longs>", () -> randomList(minRows, maxRows, () -> min), DataType.LONG, false, true)
-            );
+            addSuppliers(cases, minRows, maxRows, min + " long", DataType.LONG, () -> min);
         }
 
         long lower = Math.max(min, 1);
         long upper = Math.min(max, Long.MAX_VALUE);
         if (lower < upper) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<positive longs>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomLongBetween(lower, upper)),
-                    DataType.LONG,
-                    false,
-                    true
-                )
-            );
+            addSuppliers(cases, minRows, maxRows, "positive long", DataType.LONG, () -> ESTestCase.randomLongBetween(lower, upper));
         }
 
         long lower1 = Math.max(min, Long.MIN_VALUE);
         long upper1 = Math.min(max, -1);
         if (lower1 < upper1) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<negative longs>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomLongBetween(lower1, upper1)),
-                    DataType.LONG,
-                    false,
-                    true
-                )
-            );
+            addSuppliers(cases, minRows, maxRows, "negative long", DataType.LONG, () -> ESTestCase.randomLongBetween(lower1, upper1));
         }
 
         if (min < 0 && max > 0) {
-            cases.add(new TypedDataSupplier("<random longs>", () -> randomList(minRows, maxRows, () -> {
+            addSuppliers(cases, minRows, maxRows, "random long", DataType.LONG, () -> {
                 if (includeZero) {
                     return ESTestCase.randomLongBetween(min, max);
                 }
                 return randomBoolean() ? ESTestCase.randomLongBetween(min, -1) : ESTestCase.randomLongBetween(1, max);
-            }), DataType.LONG, false, true));
+            });
         }
 
         return cases;
@@ -156,29 +117,20 @@ public static List<TypedDataSupplier> ulongCases(int minRows, int maxRows, BigIn
 
         // Zero
         if (BigInteger.ZERO.compareTo(max) <= 0 && BigInteger.ZERO.compareTo(min) >= 0 && includeZero) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<0 unsigned longs>",
-                    () -> randomList(minRows, maxRows, () -> BigInteger.ZERO),
-                    DataType.UNSIGNED_LONG,
-                    false,
-                    true
-                )
-            );
+            addSuppliers(cases, minRows, maxRows, "0 unsigned long", DataType.UNSIGNED_LONG, () -> BigInteger.ZERO);
         }
 
         // Small values, less than Long.MAX_VALUE
         BigInteger lower1 = min.max(BigInteger.ONE);
         BigInteger upper1 = max.min(BigInteger.valueOf(Long.MAX_VALUE));
         if (lower1.compareTo(upper1) < 0) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<small unsigned longs>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomUnsignedLongBetween(lower1, upper1)),
-                    DataType.UNSIGNED_LONG,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "small unsigned long",
+                DataType.UNSIGNED_LONG,
+                () -> ESTestCase.randomUnsignedLongBetween(lower1, upper1)
             );
         }
 
@@ -186,14 +138,13 @@ public static List<TypedDataSupplier> ulongCases(int minRows, int maxRows, BigIn
         BigInteger lower2 = min.max(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE));
         BigInteger upper2 = max.min(ESTestCase.UNSIGNED_LONG_MAX);
         if (lower2.compareTo(upper2) < 0) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<big unsigned longs>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomUnsignedLongBetween(lower2, upper2)),
-                    DataType.UNSIGNED_LONG,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "big unsigned long",
+                DataType.UNSIGNED_LONG,
+                () -> ESTestCase.randomUnsignedLongBetween(lower2, upper2)
             );
         }
 
@@ -204,85 +155,77 @@ public static List<TypedDataSupplier> doubleCases(int minRows, int maxRows, doub
         List<TypedDataSupplier> cases = new ArrayList<>();
 
         if (0d <= max && 0d >= min && includeZero) {
-            cases.add(new TypedDataSupplier("<0 doubles>", () -> randomList(minRows, maxRows, () -> 0d), DataType.DOUBLE, false, true));
-            cases.add(new TypedDataSupplier("<-0 doubles>", () -> randomList(minRows, maxRows, () -> -0d), DataType.DOUBLE, false, true));
+            addSuppliers(cases, minRows, maxRows, "0 double", DataType.DOUBLE, () -> 0d);
+            addSuppliers(cases, minRows, maxRows, "-0 double", DataType.DOUBLE, () -> -0d);
         }
 
         if (max != 0d) {
-            cases.add(
-                new TypedDataSupplier("<" + max + " doubles>", () -> randomList(minRows, maxRows, () -> max), DataType.DOUBLE, false, true)
-            );
+            addSuppliers(cases, minRows, maxRows, max + " double", DataType.DOUBLE, () -> max);
         }
 
         if (min != 0d && min != max) {
-            cases.add(
-                new TypedDataSupplier("<" + min + " doubles>", () -> randomList(minRows, maxRows, () -> min), DataType.DOUBLE, false, true)
-            );
+            addSuppliers(cases, minRows, maxRows, min + " double", DataType.DOUBLE, () -> min);
         }
 
         double lower1 = Math.max(min, 0d);
         double upper1 = Math.min(max, 1d);
         if (lower1 < upper1) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<small positive doubles>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomDoubleBetween(lower1, upper1, true)),
-                    DataType.DOUBLE,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "small positive double",
+                DataType.DOUBLE,
+                () -> ESTestCase.randomDoubleBetween(lower1, upper1, true)
             );
         }
 
         double lower2 = Math.max(min, -1d);
         double upper2 = Math.min(max, 0d);
         if (lower2 < upper2) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<small negative doubles>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomDoubleBetween(lower2, upper2, true)),
-                    DataType.DOUBLE,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "small negative double",
+                DataType.DOUBLE,
+                () -> ESTestCase.randomDoubleBetween(lower2, upper2, true)
             );
         }
 
         double lower3 = Math.max(min, 1d);
         double upper3 = Math.min(max, Double.MAX_VALUE);
         if (lower3 < upper3) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<big positive doubles>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomDoubleBetween(lower3, upper3, true)),
-                    DataType.DOUBLE,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "big positive double",
+                DataType.DOUBLE,
+                () -> ESTestCase.randomDoubleBetween(lower3, upper3, true)
             );
         }
 
         double lower4 = Math.max(min, -Double.MAX_VALUE);
         double upper4 = Math.min(max, -1d);
         if (lower4 < upper4) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<big negative doubles>",
-                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomDoubleBetween(lower4, upper4, true)),
-                    DataType.DOUBLE,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "big negative double",
+                DataType.DOUBLE,
+                () -> ESTestCase.randomDoubleBetween(lower4, upper4, true)
             );
         }
 
         if (min < 0 && max > 0) {
-            cases.add(new TypedDataSupplier("<random doubles>", () -> randomList(minRows, maxRows, () -> {
+            addSuppliers(cases, minRows, maxRows, "random double", DataType.DOUBLE, () -> {
                 if (includeZero) {
                     return ESTestCase.randomDoubleBetween(min, max, true);
                 }
                 return randomBoolean() ? ESTestCase.randomDoubleBetween(min, -1, true) : ESTestCase.randomDoubleBetween(1, max, true);
-            }), DataType.DOUBLE, false, true));
+            });
         }
 
         return cases;
@@ -291,149 +234,126 @@ public static List<TypedDataSupplier> doubleCases(int minRows, int maxRows, doub
     public static List<TypedDataSupplier> dateCases(int minRows, int maxRows) {
         List<TypedDataSupplier> cases = new ArrayList<>();
 
-        cases.add(
-            new TypedDataSupplier(
-                "<1970-01-01T00:00:00Z dates>",
-                () -> randomList(minRows, maxRows, () -> 0L),
-                DataType.DATETIME,
-                false,
-                true
-            )
-        );
+        addSuppliers(cases, minRows, maxRows, "1970-01-01T00:00:00Z date", DataType.DATETIME, () -> 0L);
 
-        cases.add(
-            new TypedDataSupplier(
-                "<random dates>",
-                // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z
-                () -> randomList(minRows, maxRows, () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11)),
-                DataType.DATETIME,
-                false,
-                true
-            )
-        );
+        // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z
+        addSuppliers(cases, minRows, maxRows, "random date", DataType.DATETIME, () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11));
 
-        cases.add(
-            new TypedDataSupplier(
-                "<far future dates>",
-                // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z
-                () -> randomList(minRows, maxRows, () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE)),
-                DataType.DATETIME,
-                false,
-                true
-            )
+        // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "far future date",
+            DataType.DATETIME,
+            () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE)
         );
 
-        cases.add(
-            new TypedDataSupplier(
-                "<near the end of time dates>",
-                // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch
-                () -> randomList(minRows, maxRows, () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE)),
-                DataType.DATETIME,
-                false,
-                true
-            )
+        // Very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "near the end of time date",
+            DataType.DATETIME,
+            () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE)
         );
 
         return cases;
     }
 
     public static List<TypedDataSupplier> booleanCases(int minRows, int maxRows) {
-        return List.of(
-            new TypedDataSupplier("<true booleans>", () -> randomList(minRows, maxRows, () -> true), DataType.BOOLEAN, false, true),
-            new TypedDataSupplier("<false booleans>", () -> randomList(minRows, maxRows, () -> false), DataType.BOOLEAN, false, true),
-            new TypedDataSupplier(
-                "<random booleans>",
-                () -> randomList(minRows, maxRows, ESTestCase::randomBoolean),
-                DataType.BOOLEAN,
-                false,
-                true
-            )
-        );
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        addSuppliers(cases, minRows, maxRows, "true boolean", DataType.BOOLEAN, () -> true);
+        addSuppliers(cases, minRows, maxRows, "false boolean", DataType.BOOLEAN, () -> false);
+        addSuppliers(cases, minRows, maxRows, "random boolean", DataType.BOOLEAN, ESTestCase::randomBoolean);
+
+        return cases;
     }
 
     public static List<TypedDataSupplier> ipCases(int minRows, int maxRows) {
-        return List.of(
-            new TypedDataSupplier(
-                "<127.0.0.1 ips>",
-                () -> randomList(minRows, maxRows, () -> new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))),
-                DataType.IP,
-                false,
-                true
-            ),
-            new TypedDataSupplier(
-                "<v4 ips>",
-                () -> randomList(minRows, maxRows, () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(true)))),
-                DataType.IP,
-                false,
-                true
-            ),
-            new TypedDataSupplier(
-                "<v6 ips>",
-                () -> randomList(minRows, maxRows, () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(false)))),
-                DataType.IP,
-                false,
-                true
-            )
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "127.0.0.1 ip",
+            DataType.IP,
+            () -> new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))
+        );
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "random v4 ip",
+            DataType.IP,
+            () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(true)))
+        );
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "random v6 ip",
+            DataType.IP,
+            () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(false)))
         );
+
+        return cases;
     }
 
     public static List<TypedDataSupplier> versionCases(int minRows, int maxRows) {
-        return List.of(
-            new TypedDataSupplier(
-                "<major versions>",
-                () -> randomList(minRows, maxRows, () -> new Version(Integer.toString(ESTestCase.between(0, 100))).toBytesRef()),
-                DataType.VERSION,
-                false,
-                true
-            ),
-            new TypedDataSupplier(
-                "<major.minor versions>",
-                () -> randomList(
-                    minRows,
-                    maxRows,
-                    () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100)).toBytesRef()
-                ),
-                DataType.VERSION,
-                false,
-                true
-            ),
-            new TypedDataSupplier(
-                "<major.minor.patch versions>",
-                () -> randomList(
-                    minRows,
-                    maxRows,
-                    () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100))
-                        .toBytesRef()
-                ),
-                DataType.VERSION,
-                false,
-                true
-            )
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "major version",
+            DataType.VERSION,
+            () -> new Version(Integer.toString(ESTestCase.between(0, 100))).toBytesRef()
         );
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "major.minor version",
+            DataType.VERSION,
+            () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100)).toBytesRef()
+        );
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "major.minor.patch version",
+            DataType.VERSION,
+            () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100)).toBytesRef()
+        );
+
+        return cases;
     }
 
     public static List<TypedDataSupplier> geoPointCases(int minRows, int maxRows, boolean withAltitude) {
         List<TypedDataSupplier> cases = new ArrayList<>();
 
-        cases.add(
-            new TypedDataSupplier(
-                "<no alt geo_points>",
-                () -> randomList(minRows, maxRows, () -> GEO.asWkb(GeometryTestUtils.randomPoint(false))),
-                DataType.GEO_POINT,
-                false,
-                true
-            )
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "<no alt geo_point>",
+            DataType.GEO_POINT,
+            () -> GEO.asWkb(GeometryTestUtils.randomPoint(false))
         );
 
         if (withAltitude) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<with alt geo_points>",
-                    () -> randomList(minRows, maxRows, () -> GEO.asWkb(GeometryTestUtils.randomPoint(false))),
-                    DataType.GEO_POINT,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "<with alt geo_point>",
+                DataType.GEO_POINT,
+                () -> GEO.asWkb(GeometryTestUtils.randomPoint(true))
             );
         }
 
@@ -443,25 +363,23 @@ public static List<TypedDataSupplier> geoPointCases(int minRows, int maxRows, bo
     public static List<TypedDataSupplier> cartesianPointCases(int minRows, int maxRows, boolean withAltitude) {
         List<TypedDataSupplier> cases = new ArrayList<>();
 
-        cases.add(
-            new TypedDataSupplier(
-                "<no alt cartesian_points>",
-                () -> randomList(minRows, maxRows, () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(false))),
-                DataType.CARTESIAN_POINT,
-                false,
-                true
-            )
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "<no alt cartesian_point>",
+            DataType.CARTESIAN_POINT,
+            () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(false))
         );
 
         if (withAltitude) {
-            cases.add(
-                new TypedDataSupplier(
-                    "<with alt cartesian_points>",
-                    () -> randomList(minRows, maxRows, () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(true))),
-                    DataType.CARTESIAN_POINT,
-                    false,
-                    true
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                maxRows,
+                "<with alt cartesian_point>",
+                DataType.CARTESIAN_POINT,
+                () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(true))
             );
         }
 
@@ -471,59 +389,64 @@ public static List<TypedDataSupplier> cartesianPointCases(int minRows, int maxRo
     public static List<TypedDataSupplier> stringCases(int minRows, int maxRows, DataType type) {
         List<TypedDataSupplier> cases = new ArrayList<>();
 
-        cases.addAll(
-            List.of(
-                new TypedDataSupplier(
-                    "<empty " + type + "s>",
-                    () -> randomList(minRows, maxRows, () -> new BytesRef("")),
-                    type,
-                    false,
-                    true
-                ),
-                new TypedDataSupplier(
-                    "<short alpha " + type + "s>",
-                    () -> randomList(minRows, maxRows, () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(1, 30))),
-                    type,
-                    false,
-                    true
-                ),
-                new TypedDataSupplier(
-                    "<short unicode " + type + "s>",
-                    () -> randomList(minRows, maxRows, () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(1, 30))),
-                    type,
-                    false,
-                    true
-                )
-            )
+        addSuppliers(cases, minRows, maxRows, "empty " + type, type, () -> new BytesRef(""));
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "short alpha " + type,
+            type,
+            () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(1, 30))
+        );
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "short unicode " + type,
+            type,
+            () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(1, 30))
         );
 
         if (minRows <= 100) {
             var longStringsMaxRows = Math.min(maxRows, 100);
 
-            cases.addAll(
-                List.of(
-                    new TypedDataSupplier(
-                        "<long alpha " + type + "s>",
-                        () -> randomList(minRows, longStringsMaxRows, () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(300, 1000))),
-                        type,
-                        false,
-                        true
-                    ),
-                    new TypedDataSupplier(
-                        "<long unicode " + type + "s>",
-                        () -> randomList(
-                            minRows,
-                            longStringsMaxRows,
-                            () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(300, 1000))
-                        ),
-                        type,
-                        false,
-                        true
-                    )
-                )
+            addSuppliers(
+                cases,
+                minRows,
+                longStringsMaxRows,
+                "long alpha " + type,
+                type,
+                () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(300, 1000))
+            );
+            addSuppliers(
+                cases,
+                minRows,
+                longStringsMaxRows,
+                "long unicode " + type,
+                type,
+                () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(300, 1000))
             );
         }
 
         return cases;
     }
+
+    private static <T> void addSuppliers(
+        List<TypedDataSupplier> cases,
+        int minRows,
+        int maxRows,
+        String name,
+        DataType type,
+        Supplier<T> valueSupplier
+    ) {
+        if (minRows <= 1 && maxRows >= 1) {
+            cases.add(new TypedDataSupplier("<single " + name + ">", () -> randomList(1, 1, valueSupplier), type, false, true));
+        }
+
+        if (maxRows > 1) {
+            cases.add(
+                new TypedDataSupplier("<" + name + "s>", () -> randomList(Math.max(2, minRows), maxRows, valueSupplier), type, false, true)
+            );
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
index b3942a71edadb..2ba175657b6c2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
-import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
 import org.elasticsearch.xpack.versionfield.Version;
 import org.hamcrest.Matcher;
 
@@ -91,7 +90,7 @@ public static List<TestCaseSupplier> stringCases(
         List<TypedDataSupplier> lhsSuppliers = new ArrayList<>();
         List<TypedDataSupplier> rhsSuppliers = new ArrayList<>();
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType type : AbstractConvertFunction.STRING_TYPES) {
+        for (DataType type : DataType.stringTypes()) {
             lhsSuppliers.addAll(stringCases(type));
             rhsSuppliers.addAll(stringCases(type));
             casesCrossProduct(
@@ -760,7 +759,7 @@ public static void forUnaryStrings(
         Function<BytesRef, Object> expectedValue,
         Function<BytesRef, List<String>> expectedWarnings
     ) {
-        for (DataType type : AbstractConvertFunction.STRING_TYPES) {
+        for (DataType type : DataType.stringTypes()) {
             unary(
                 suppliers,
                 expectedEvaluatorToString,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java
index 80737dac1aa58..ac599c7ff05f8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java
@@ -64,25 +64,27 @@ protected Expression build(Source source, List<Expression> args) {
     private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
         return new TestCaseSupplier(List.of(fieldSupplier.type()), () -> {
             var fieldTypedData = fieldSupplier.get();
+            var fieldData = fieldTypedData.multiRowData();
 
-            Object expected = switch (fieldTypedData.type().widenSmallNumeric()) {
-                case INTEGER -> fieldTypedData.multiRowData()
-                    .stream()
-                    .map(v -> (Integer) v)
-                    .collect(Collectors.summarizingInt(Integer::intValue))
-                    .getAverage();
-                case LONG -> fieldTypedData.multiRowData()
-                    .stream()
-                    .map(v -> (Long) v)
-                    .collect(Collectors.summarizingLong(Long::longValue))
-                    .getAverage();
-                case DOUBLE -> fieldTypedData.multiRowData()
-                    .stream()
-                    .map(v -> (Double) v)
-                    .collect(Collectors.summarizingDouble(Double::doubleValue))
-                    .getAverage();
-                default -> throw new IllegalStateException("Unexpected value: " + fieldTypedData.type());
-            };
+            Object expected = null;
+
+            if (fieldData.size() == 1) {
+                // For single elements, we directly return them to avoid precision issues
+                expected = ((Number) fieldData.get(0)).doubleValue();
+            } else if (fieldData.size() > 1) {
+                expected = switch (fieldTypedData.type().widenSmallNumeric()) {
+                    case INTEGER -> fieldData.stream()
+                        .map(v -> (Integer) v)
+                        .collect(Collectors.summarizingInt(Integer::intValue))
+                        .getAverage();
+                    case LONG -> fieldData.stream().map(v -> (Long) v).collect(Collectors.summarizingLong(Long::longValue)).getAverage();
+                    case DOUBLE -> fieldData.stream()
+                        .map(v -> (Double) v)
+                        .collect(Collectors.summarizingDouble(Double::doubleValue))
+                        .getAverage();
+                    default -> throw new IllegalStateException("Unexpected value: " + fieldTypedData.type());
+                };
+            }
 
             return new TestCaseSupplier.TestCase(
                 List.of(fieldTypedData),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java
index 94b2a81b308d7..ea7c480817317 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java
@@ -36,4 +36,9 @@ protected Rate mutateInstance(Rate instance) throws IOException {
         }
         return new Rate(source, field, timestamp, unit);
     }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java
index b79252c694084..15ea029a05554 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java
@@ -22,13 +22,15 @@
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
 
-import java.nio.ByteOrder;
 import java.util.List;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
-import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.closeTo;
 
 @FunctionName("st_centroid_agg")
 public class SpatialCentroidTests extends AbstractAggregationTestCase {
@@ -74,16 +76,58 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier
                 count++;
             }
 
-            var expected = new BytesRef(
-                WellKnownBinary.toWKB(new Point(xSum.value() / count, ySum.value() / count), ByteOrder.LITTLE_ENDIAN)
-            );
+            var expectedX = xSum.value() / count;
+            var expectedY = ySum.value() / count;
 
             return new TestCaseSupplier.TestCase(
                 List.of(fieldTypedData),
                 "SpatialCentroid[field=Attribute[channel=0]]",
                 fieldTypedData.type(),
-                equalTo(expected)
+                centroidMatches(expectedX, expectedY, 1e-14)
             );
         });
     }
+
+    @SuppressWarnings("SameParameterValue")
+    private static Matcher<BytesRef> centroidMatches(double x, double y, double error) {
+        return new TestCentroidMatcher(x, y, error);
+    }
+
+    private static class TestCentroidMatcher extends BaseMatcher<BytesRef> {
+        private final double x;
+        private final double y;
+        private final Matcher<Double> mx;
+        private final Matcher<Double> my;
+
+        private TestCentroidMatcher(double x, double y, double error) {
+            this.x = x;
+            this.y = y;
+            this.mx = closeTo(x, error);
+            this.my = closeTo(y, error);
+        }
+
+        @Override
+        public boolean matches(Object item) {
+            if (item instanceof BytesRef wkb) {
+                var point = (Point) WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length);
+                return mx.matches(point.getX()) && my.matches(point.getY());
+            }
+            return false;
+        }
+
+        @Override
+        public void describeMismatch(Object item, Description description) {
+            if (item instanceof BytesRef wkb) {
+                var point = (Point) WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length);
+                description.appendText("was ").appendValue(point);
+            } else {
+                description.appendText("was ").appendValue(item);
+            }
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description.appendValue("    POINT (" + x + " " + y + ")");
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java
index 82bf57d1a194e..e74b26c87c84f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java
@@ -36,4 +36,9 @@ protected Top mutateInstance(Top instance) throws IOException {
         }
         return new Top(source, field, limit, order);
     }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java
new file mode 100644
index 0000000000000..967b4d854c325
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.fulltext;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.FunctionName;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+
+@FunctionName("match")
+public class MatchTests extends AbstractFunctionTestCase {
+
+    public MatchTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        Set<DataType> supported = Set.of(DataType.KEYWORD, DataType.TEXT);
+        List<Set<DataType>> supportedPerPosition = List.of(supported, supported);
+        List<TestCaseSupplier> suppliers = new LinkedList<>();
+        for (DataType fieldType : DataType.stringTypes()) {
+            for (DataType queryType : DataType.stringTypes()) {
+                suppliers.add(
+                    new TestCaseSupplier(
+                        "<" + fieldType + "-ES field, " + queryType + ">",
+                        List.of(fieldType, queryType),
+                        () -> testCase(fieldType, randomIdentifier(), queryType, randomAlphaOfLengthBetween(1, 10), equalTo(true))
+                    )
+                );
+                suppliers.add(
+                    new TestCaseSupplier(
+                        "<" + fieldType + "-non ES field, " + queryType + ">",
+                        List.of(fieldType, queryType),
+                        typeErrorSupplier(true, supportedPerPosition, List.of(fieldType, queryType), MatchTests::matchTypeErrorSupplier)
+                    )
+                );
+            }
+        }
+        List<TestCaseSupplier> errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string");
+        // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests
+        return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList());
+    }
+
+    private static String matchTypeErrorSupplier(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types) {
+        return "[] cannot operate on [" + types.getFirst().typeName() + "], which is not a field from an index mapping";
+    }
+
+    private static TestCaseSupplier.TestCase testCase(
+        DataType fieldType,
+        String field,
+        DataType queryType,
+        String query,
+        Matcher<Boolean> matcher
+    ) {
+        return new TestCaseSupplier.TestCase(
+            List.of(
+                new TestCaseSupplier.TypedData(
+                    new FieldExpression(field, List.of(new FieldExpression.FieldValue(field))),
+                    fieldType,
+                    "field"
+                ),
+                new TestCaseSupplier.TypedData(new BytesRef(query), queryType, "query")
+            ),
+            "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
+            DataType.BOOLEAN,
+            matcher
+        );
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Match(source, args.get(0), args.get(1));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java
similarity index 81%
rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java
index 37e16a2499cd9..b4b4ebcaacde6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java
@@ -18,32 +18,24 @@
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
-import org.junit.BeforeClass;
 
-import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.QSTR_FUNCTION;
 import static org.hamcrest.Matchers.equalTo;
 
 @FunctionName("qstr")
-public class QueryStringFunctionTests extends AbstractFunctionTestCase {
+public class QueryStringTests extends AbstractFunctionTestCase {
 
-    @BeforeClass
-    public static void checkFunctionEnabled() {
-        assumeTrue("QSTR capability should be enabled ", QSTR_FUNCTION.isEnabled());
-    }
-
-    public QueryStringFunctionTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+    public QueryStringTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         this.testCase = testCaseSupplier.get();
     }
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new LinkedList<>();
-        for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
+        for (DataType strType : DataType.stringTypes()) {
             suppliers.add(
                 new TestCaseSupplier(
                     "<" + strType + ">",
@@ -77,6 +69,6 @@ private static TestCaseSupplier.TestCase testCase(DataType strType, String str,
 
     @Override
     protected Expression build(Source source, List<Expression> args) {
-        return new QueryStringFunction(source, args.get(0));
+        return new QueryString(source, args.get(0));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
index 9d0d9c3da30a8..fbb7c691b1d94 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
@@ -46,6 +46,7 @@ public class CaseTests extends AbstractScalarFunctionTestCase {
             DataType.TEXT,
             DataType.BOOLEAN,
             DataType.DATETIME,
+            DataType.DATE_NANOS,
             DataType.DOUBLE,
             DataType.INTEGER,
             DataType.LONG,
@@ -59,7 +60,7 @@ public class CaseTests extends AbstractScalarFunctionTestCase {
             DataType.NULL
         ).collect(Collectors.toList());
         if (Build.current().isSnapshot()) {
-            t.addAll(DataType.UNDER_CONSTRUCTION.keySet());
+            t.add(DataType.DATE_NANOS);
         }
         TYPES = unmodifiableList(t);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java
index 311e3e3d89149..07d6ae34dc1e7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java
@@ -115,6 +115,21 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
+        suppliers.add(
+            new TestCaseSupplier(
+                "(a, b)",
+                List.of(DataType.DATE_NANOS, DataType.DATE_NANOS),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(1727877348000123456L, DataType.DATE_NANOS, "a"),
+                        new TestCaseSupplier.TypedData(1727790948000987654L, DataType.DATE_NANOS, "b")
+                    ),
+                    "GreatestLongEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]",
+                    DataType.DATE_NANOS,
+                    equalTo(1727877348000123456L)
+                )
+            )
+        );
         return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java
index 69842fde90312..d95cc79dd22e0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java
@@ -114,6 +114,21 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
+        suppliers.add(
+            new TestCaseSupplier(
+                "(a, b)",
+                List.of(DataType.DATE_NANOS, DataType.DATE_NANOS),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(1727877348000123456L, DataType.DATE_NANOS, "a"),
+                        new TestCaseSupplier.TypedData(1727790948000987654L, DataType.DATE_NANOS, "b")
+                    ),
+                    "LeastLongEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]",
+                    DataType.DATE_NANOS,
+                    equalTo(1727790948000987654L)
+                )
+            )
+        );
         return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java
index e91a5cc1ebca4..485073d1a91d2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java
@@ -11,6 +11,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -29,6 +30,9 @@ public ToDateNanosTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> te
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
+        if (EsqlCapabilities.Cap.TO_DATE_NANOS.isEnabled() == false) {
+            return List.of();
+        }
         final String read = "Attribute[channel=0]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
index 7799c3c756f23..2852b92ba156e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
@@ -134,9 +134,9 @@ public static Iterable<Object[]> parameters() {
             "ToDatetimeFromStringEvaluator[field=" + read + "]",
             List.of(
                 new TestCaseSupplier.TypedDataSupplier(
-                    "<date string before 0001-01-01T00:00:00.000Z>",
-                    // millis before "0001-01-01T00:00:00.000Z"
-                    () -> new BytesRef(randomDateString(Long.MIN_VALUE, -62135596800001L)),
+                    "<date string before -9999-12-31T23:59:59.999Z>",
+                    // millis before "-9999-12-31T23:59:59.999Z"
+                    () -> new BytesRef(randomDateString(Long.MIN_VALUE, -377736739200000L)),
                     DataType.KEYWORD
                 )
             ),
@@ -154,8 +154,8 @@ public static Iterable<Object[]> parameters() {
             "ToDatetimeFromStringEvaluator[field=" + read + "]",
             List.of(
                 new TestCaseSupplier.TypedDataSupplier(
-                    "<date string before 0001-01-01T00:00:00.000Z>",
-                    // millis before "0001-01-01T00:00:00.000Z"
+                    "<date string after 9999-12-31T23:59:59.999Z>",
+                    // millis after "9999-12-31T23:59:59.999Z"
                     () -> new BytesRef(randomDateString(253402300800000L, Long.MAX_VALUE)),
                     DataType.KEYWORD
                 )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
index 46a8086f9479c..57f11331818dc 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
@@ -49,7 +49,7 @@ public static Iterable<Object[]> parameters() {
         );
 
         // But strings that are shaped like versions do parse to valid versions
-        for (DataType inputType : AbstractConvertFunction.STRING_TYPES) {
+        for (DataType inputType : DataType.stringTypes()) {
             TestCaseSupplier.unary(
                 suppliers,
                 read,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotSerializationTests.java
new file mode 100644
index 0000000000000..5c2e84fcba8e0
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotSerializationTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests;
+
+import java.io.IOException;
+
+public class HypotSerializationTests extends AbstractExpressionSerializationTests<Hypot> {
+
+    @Override
+    protected Hypot createTestInstance() {
+        Source source = randomSource();
+        Expression n1 = randomChild();
+        Expression n2 = randomChild();
+        return new Hypot(source, n1, n2);
+    }
+
+    @Override
+    protected Hypot mutateInstance(Hypot instance) throws IOException {
+        Source source = instance.source();
+        Expression n1 = instance.n1();
+        Expression n2 = instance.n2();
+        if (randomBoolean()) {
+            n1 = randomValueOtherThan(n1, AbstractUnaryScalarSerializationTests::randomChild);
+        } else {
+            n2 = randomValueOtherThan(n2, AbstractUnaryScalarSerializationTests::randomChild);
+        }
+        return new Hypot(source, n1, n2);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotTests.java
new file mode 100644
index 0000000000000..0161abc2b9560
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotTests.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+public class HypotTests extends AbstractScalarFunctionTestCase {
+    public HypotTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> suppliers = TestCaseSupplier.forBinaryCastingToDouble(
+            "HypotEvaluator",
+            "n1",
+            "n2",
+            Math::hypot,
+            Double.NEGATIVE_INFINITY,
+            Double.POSITIVE_INFINITY,
+            Double.NEGATIVE_INFINITY,
+            Double.POSITIVE_INFINITY,
+            List.of()
+        );
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Hypot(source, args.get(0), args.get(1));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
index d8d3b607efcc0..f3b44274f3ade 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
@@ -39,6 +39,7 @@ public static Iterable<Object[]> parameters() {
         booleans(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
         bytesRefs(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
         dateTimes(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Long::valueOf)));
+        dateNanos(cases, "mv_dedupe", "MvDedupe", DataType.DATE_NANOS, (size, values) -> getMatcher(values.mapToObj(Long::valueOf)));
         doubles(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Double::valueOf)));
         ints(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Integer::valueOf)));
         longs(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Long::valueOf)));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java
index 29cc959e6a943..0a419d44e3448 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.MultivalueTestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
@@ -28,6 +29,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
@@ -375,27 +377,25 @@ private static TestCaseSupplier makeSupplier(
                 var values = (List<Number>) fieldTypedData.data();
                 var percentile = ((Number) percentileTypedData.data()).doubleValue();
 
-                var expected = calculatePercentile(values, percentile);
+                var expectedMatcher = makePercentileMatcher(values, percentile);
 
                 return new TestCaseSupplier.TestCase(
                     List.of(fieldTypedData, percentileTypedData),
                     evaluatorString(fieldSupplier.type(), percentileSupplier.type()),
                     fieldSupplier.type(),
-                    expected instanceof Double expectedDouble
-                        ? closeTo(expectedDouble, Math.abs(expectedDouble * 0.0000001))
-                        : equalTo(expected)
+                    expectedMatcher
                 );
             }
         );
     }
 
-    private static Number calculatePercentile(List<Number> rawValues, double percentile) {
+    private static Matcher<?> makePercentileMatcher(List<Number> rawValues, double percentile) {
         if (rawValues.isEmpty() || percentile < 0 || percentile > 100) {
-            return null;
+            return nullValue();
         }
 
         if (rawValues.size() == 1) {
-            return rawValues.get(0);
+            return equalTo(rawValues.get(0));
         }
 
         int valueCount = rawValues.size();
@@ -407,49 +407,62 @@ private static Number calculatePercentile(List<Number> rawValues, double percent
 
         if (rawValues.get(0) instanceof Integer) {
             var values = rawValues.stream().mapToInt(Number::intValue).sorted().toArray();
+            int expected;
 
             if (percentile == 0) {
-                return values[0];
+                expected = values[0];
             } else if (percentile == 100) {
-                return values[valueCount - 1];
+                expected = values[valueCount - 1];
             } else {
                 assert lowerIndex >= 0 && upperIndex < valueCount;
                 var difference = (long) values[upperIndex] - values[lowerIndex];
-                return values[lowerIndex] + (int) (fraction * difference);
+                expected = values[lowerIndex] + (int) (fraction * difference);
             }
+
+            return equalTo(expected);
         }
 
         if (rawValues.get(0) instanceof Long) {
             var values = rawValues.stream().mapToLong(Number::longValue).sorted().toArray();
+            long expected;
 
             if (percentile == 0) {
-                return values[0];
+                expected = values[0];
             } else if (percentile == 100) {
-                return values[valueCount - 1];
+                expected = values[valueCount - 1];
             } else {
                 assert lowerIndex >= 0 && upperIndex < valueCount;
-                return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).longValue();
+                expected = calculatePercentile(fraction, BigDecimal.valueOf(values[lowerIndex]), BigDecimal.valueOf(values[upperIndex]))
+                    .longValue();
             }
+
+            // Double*bigLong may lose precision, we allow a small range
+            return anyOf(equalTo(Math.min(expected, expected - 1)), equalTo(expected), equalTo(Math.max(expected, expected + 1)));
         }
 
         if (rawValues.get(0) instanceof Double) {
             var values = rawValues.stream().mapToDouble(Number::doubleValue).sorted().toArray();
+            double expected;
 
             if (percentile == 0) {
-                return values[0];
+                expected = values[0];
             } else if (percentile == 100) {
-                return values[valueCount - 1];
+                expected = values[valueCount - 1];
             } else {
                 assert lowerIndex >= 0 && upperIndex < valueCount;
-                return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).doubleValue();
+                expected = calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex]))
+                    .doubleValue();
             }
+
+            return closeTo(expected, Math.abs(expected * 0.0000001));
         }
 
         throw new IllegalArgumentException("Unsupported type: " + rawValues.get(0).getClass());
     }
 
     private static BigDecimal calculatePercentile(double fraction, BigDecimal lowerValue, BigDecimal upperValue) {
-        return lowerValue.add(new BigDecimal(fraction).multiply(upperValue.subtract(lowerValue)));
+        var difference = upperValue.subtract(lowerValue);
+        return lowerValue.add(new BigDecimal(fraction).multiply(difference));
     }
 
     private static TestCaseSupplier.TypedData percentileWithType(Number value, DataType type) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
index e5bac422805af..859c79090d62f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
@@ -182,6 +182,23 @@ private static void longs(List<TestCaseSupplier> suppliers) {
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DATE_NANOS, DataType.INTEGER, DataType.INTEGER), () -> {
+            List<Long> field = randomList(1, 10, () -> randomLong());
+            int length = field.size();
+            int start = randomIntBetween(0, length - 1);
+            int end = randomIntBetween(start, length - 1);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field, DataType.DATE_NANOS, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
+                ),
+                "MvSliceLongEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
+                DataType.DATE_NANOS,
+                equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
+            );
+        }));
     }
 
     private static void doubles(List<TestCaseSupplier> suppliers) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
index d07ed2aeae887..63f538059dddf 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
@@ -110,6 +110,20 @@ private static void longs(List<TestCaseSupplier> suppliers) {
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
         }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DATE_NANOS, DataType.KEYWORD), () -> {
+            List<Long> field = randomList(1, 10, () -> randomLong());
+            BytesRef order = new BytesRef("DESC");
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field, DataType.DATE_NANOS, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
+                ),
+                "MvSortLong[field=Attribute[channel=0], order=false]",
+                DataType.DATE_NANOS,
+                equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
+            );
+        }));
     }
 
     private static void doubles(List<TestCaseSupplier> suppliers) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
index c9b6de64e079d..797c99992815e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
@@ -96,6 +96,19 @@ public static Iterable<Object[]> parameters() {
                 equalTo(firstDate == null ? secondDate : firstDate)
             );
         }));
+        noNullsSuppliers.add(new TestCaseSupplier(List.of(DataType.DATE_NANOS, DataType.DATE_NANOS), () -> {
+            Long firstDate = randomBoolean() ? null : randomNonNegativeLong();
+            Long secondDate = randomNonNegativeLong();
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(firstDate, DataType.DATE_NANOS, "first"),
+                    new TestCaseSupplier.TypedData(secondDate, DataType.DATE_NANOS, "second")
+                ),
+                "CoalesceEvaluator[values=[Attribute[channel=0], Attribute[channel=1]]]",
+                DataType.DATE_NANOS,
+                equalTo(firstDate == null ? secondDate : firstDate)
+            );
+        }));
 
         List<TestCaseSupplier> suppliers = new ArrayList<>(noNullsSuppliers);
         for (TestCaseSupplier s : noNullsSuppliers) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java
index 6794de80f7433..c93d871ca2b8c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java
@@ -267,19 +267,10 @@ private static DataType pickSpatialType(DataType leftType, DataType rightType) {
 
     private static Matcher<String> spatialEvaluatorString(DataType leftType, DataType rightType) {
         String crsType = isSpatialGeo(pickSpatialType(leftType, rightType)) ? "Geo" : "Cartesian";
-        String paramSuffix = paramSuffix();
-        String channels = channelsText("left" + paramSuffix, "right" + paramSuffix);
+        String channels = channelsText("left", "right");
         return equalTo(getFunctionClassName() + crsType + "SourceAndSourceEvaluator[" + channels + "]");
     }
 
-    private static String paramSuffix() {
-        try {
-            return getSpatialRelatesFunctionClass().getSimpleName().contains("Distance") ? "Value" : "";
-        } catch (ClassNotFoundException e) {
-            return "";
-        }
-    }
-
     private static String channelsText(String... args) {
         return IntStream.range(0, args.length).mapToObj(i -> args[i] + "=Attribute[channel=" + i + "]").collect(Collectors.joining(", "));
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
index f77a892d8682e..d069f7ffe2298 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
@@ -21,7 +21,7 @@
 public abstract class AbstractTrimTests extends AbstractScalarFunctionTestCase {
     static Iterable<Object[]> parameters(String name, boolean trimLeading, boolean trimTrailing) {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType type : strings()) {
+        for (DataType type : DataType.stringTypes()) {
             suppliers.add(new TestCaseSupplier("no whitespace/" + type, List.of(type), () -> {
                 String text = randomAlphaOfLength(8);
                 return testCase(name, type, text, text);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
index bbe92ae4a6618..2ad953c9296b7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
@@ -58,9 +58,7 @@ public static Iterable<Object[]> parameters() {
                 if (rhs == DataType.NULL || DataType.isRepresentable(rhs) == false) {
                     continue;
                 }
-                boolean lhsIsString = lhs == DataType.KEYWORD || lhs == DataType.TEXT;
-                boolean rhsIsString = rhs == DataType.KEYWORD || rhs == DataType.TEXT;
-                if (lhsIsString && rhsIsString) {
+                if (DataType.isString(lhs) && DataType.isString(rhs)) {
                     continue;
                 }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
index 6d086e2626cb6..1b2e9c41cb25c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
 
-import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.function.Supplier;
@@ -33,8 +32,8 @@ public EndsWithTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testC
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new LinkedList<>();
-        for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
-            for (DataType suffixType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
+        for (DataType strType : DataType.stringTypes()) {
+            for (DataType suffixType : DataType.stringTypes()) {
                 suppliers.add(
                     new TestCaseSupplier(
                         "<" + strType + ">, empty <" + suffixType + ">",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
index 207125bed2a19..a10f97c45aa04 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
@@ -35,13 +35,11 @@ public LocateTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCas
         this.testCase = testCaseSupplier.get();
     }
 
-    private static final DataType[] STRING_TYPES = new DataType[] { DataType.KEYWORD, DataType.TEXT };
-
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType strType : STRING_TYPES) {
-            for (DataType substrType : STRING_TYPES) {
+        for (DataType strType : DataType.stringTypes()) {
+            for (DataType substrType : DataType.stringTypes()) {
                 suppliers.add(
                     supplier(
                         "",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
index 5a34d850cffe3..dab2fca212ff4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
@@ -69,7 +69,7 @@ static Iterable<Object[]> parameters(Function<String, String> escapeString, Supp
         casesForString(cases, "6 bytes, 2 code points", () -> "❗️", false, escapeString, optionalPattern);
         casesForString(cases, "100 random code points", () -> randomUnicodeOfCodepointLength(100), true, escapeString, optionalPattern);
         for (DataType type : DataType.types()) {
-            if (type == DataType.KEYWORD || type == DataType.TEXT || type == DataType.NULL) {
+            if (DataType.isString(type) || type == DataType.NULL) {
                 continue;
             }
             if (DataType.isRepresentable(type) == false) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
index 8bc8cf3184a75..60ed3b05ad642 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -32,8 +31,8 @@ public StartsWithTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> tes
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
-            for (DataType prefixType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
+        for (DataType strType : DataType.stringTypes()) {
+            for (DataType prefixType : DataType.stringTypes()) {
                 suppliers.add(new TestCaseSupplier(List.of(strType, prefixType), () -> {
                     String str = randomAlphaOfLength(5);
                     String prefix = randomAlphaOfLength(5);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java
index c145d770409da..e735ba83168bb 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java
@@ -82,7 +82,7 @@ public class TextFormatterTests extends ESTestCase {
         null,
         randomBoolean(),
         randomBoolean(),
-        new EsqlExecutionInfo()
+        new EsqlExecutionInfo(randomBoolean())
     );
 
     TextFormatter formatter = new TextFormatter(esqlResponse);
@@ -157,7 +157,7 @@ public void testFormatWithoutHeader() {
             null,
             randomBoolean(),
             randomBoolean(),
-            new EsqlExecutionInfo()
+            new EsqlExecutionInfo(randomBoolean())
         );
 
         String[] result = getTextBodyContent(new TextFormatter(response).format(false)).split("\n");
@@ -198,7 +198,7 @@ public void testVeryLongPadding() {
                         null,
                         randomBoolean(),
                         randomBoolean(),
-                        new EsqlExecutionInfo()
+                        new EsqlExecutionInfo(randomBoolean())
                     )
                 ).format(false)
             )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
index 687b83370f571..82dd5a88ffaf1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
@@ -182,4 +182,51 @@ private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) t
             assertThat(ByteSizeValue.ofBytes(out.bytes().length()), byteSizeEquals(expected));
         }
     }
+
+    public static EsIndex deeplyNestedIndex(int depth, int childrenPerLevel) {
+        String rootFieldName = "root";
+        Map<String, EsField> fields = Map.of(rootFieldName, fieldWithRecursiveChildren(depth, childrenPerLevel, rootFieldName));
+
+        return new EsIndex("deeply-nested", fields);
+    }
+
+    private static EsField fieldWithRecursiveChildren(int depth, int childrenPerLevel, String name) {
+        assert depth >= 1;
+
+        Map<String, EsField> children = new TreeMap<>();
+        String childName;
+        if (depth == 1) {
+            for (int i = 0; i < childrenPerLevel; i++) {
+                childName = "leaf" + i;
+                children.put(childName, new EsField(childName, DataType.KEYWORD, Map.of(), true));
+            }
+        } else {
+            for (int i = 0; i < childrenPerLevel; i++) {
+                childName = "level" + depth + "child" + i;
+                children.put(childName, fieldWithRecursiveChildren(depth - 1, childrenPerLevel, childName));
+            }
+        }
+
+        return new EsField(name, DataType.OBJECT, children, false);
+    }
+
+    /**
+     * Test de-/serialization and size on the wire for an index that has multiple levels of children:
+     * A single root with 9 children, each of which has 9 children etc. 6 levels deep.
+     */
+    public void testDeeplyNestedFields() throws IOException {
+        ByteSizeValue expectedSize = ByteSizeValue.ofBytes(9425494);
+        /*
+         * History:
+         *  9425494b - string serialization #112929
+         */
+
+        int depth = 6;
+        int childrenPerLevel = 9;
+
+        try (BytesStreamOutput out = new BytesStreamOutput(); var pso = new PlanStreamOutput(out, null)) {
+            deeplyNestedIndex(depth, childrenPerLevel).writeTo(pso);
+            assertThat(ByteSizeValue.ofBytes(out.bytes().length()), byteSizeEquals(expectedSize));
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
index 33252b9dbaaa3..d3e1710a715af 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.Column;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
@@ -44,7 +43,6 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class PlanStreamOutputTests extends ESTestCase {
@@ -118,26 +116,13 @@ public void testWriteAttributeMultipleTimes() throws IOException {
             for (int i = 0; i < occurrences; i++) {
                 planStream.writeNamedWriteable(attribute);
             }
-            int depth = 0;
-            Attribute parent = attribute;
-            while (parent != null) {
-                depth++;
-                parent = parent instanceof FieldAttribute f ? f.parent() : null;
-            }
-            assertThat(planStream.cachedAttributes.size(), is(depth));
+            assertThat(planStream.cachedAttributes.size(), is(1));
             try (PlanStreamInput in = new PlanStreamInput(out.bytes().streamInput(), REGISTRY, configuration)) {
                 Attribute first = in.readNamedWriteable(Attribute.class);
                 for (int i = 1; i < occurrences; i++) {
                     Attribute next = in.readNamedWriteable(Attribute.class);
                     assertThat(first, sameInstance(next));
                 }
-                for (int i = 0; i < depth; i++) {
-                    assertThat(first, equalTo(attribute));
-                    first = first instanceof FieldAttribute f ? f.parent() : null;
-                    attribute = attribute instanceof FieldAttribute f ? f.parent() : null;
-                }
-                assertThat(first, is(nullValue()));
-                assertThat(attribute, is(nullValue()));
             }
         }
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
index c2779b7dbc46d..3436502610d62 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
@@ -21,11 +21,11 @@
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.EsqlTestUtils.TestSearchStats;
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
@@ -50,7 +50,6 @@
 import org.elasticsearch.xpack.esql.plan.physical.EvalExec;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec;
 import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
-import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
 import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -145,10 +144,14 @@ private Analyzer makeAnalyzer(String mappingFileName, EnrichResolution enrichRes
 
         return new Analyzer(
             new AnalyzerContext(config, new EsqlFunctionRegistry(), getIndexResult, enrichResolution),
-            new Verifier(new Metrics())
+            new Verifier(new Metrics(new EsqlFunctionRegistry()))
         );
     }
 
+    private Analyzer makeAnalyzer(String mappingFileName) {
+        return makeAnalyzer(mappingFileName, new EnrichResolution());
+    }
+
     /**
      * Expects
      * LimitExec[1000[INTEGER]]
@@ -385,7 +388,6 @@ public void testMultiCountAllWithFilter() {
      *       \_EsQueryExec[test], indexMode[standard], query[{"query_string":{"query":"last_name: Smith","fields":[]}}]
      */
     public void testQueryStringFunction() {
-        assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
         var plan = plannerOptimizer.plan("""
             from test
             | where qstr("last_name: Smith")
@@ -414,7 +416,6 @@ public void testQueryStringFunction() {
      *        "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324]
      */
     public void testQueryStringFunctionConjunctionWhereOperands() {
-        assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
         String queryText = """
             from test
             | where qstr("last_name: Smith") and emp_no > 10010
@@ -438,20 +439,56 @@ public void testQueryStringFunctionConjunctionWhereOperands() {
     /**
      * Expecting
      * LimitExec[1000[INTEGER]]
-     * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n
-     * ame{f}#7, long_noidx{f}#12, salary{f}#8],false]
-     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n
-     * ame{f}#7, long_noidx{f}#12, salary{f}#8]]
-     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen]
-     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":{"should":[{"query_string":{"query":"last_name: Smith","fields":[]}},
-     *       {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@2:37"}}],
-     *       "boost":1.0}}][_doc{f}#13], limit[1000], sort[] estimatedRowSize[324]
+     * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_
+     * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16],
+     * false]
+     *   \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_
+     * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16]
+     *     \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{
+     *       "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}},
+     *       "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354]
      */
-    public void testQueryStringFunctionDisjunctionWhereClauses() {
-        assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+    public void testQueryStringFunctionWithFunctionsPushedToLucene() {
         String queryText = """
             from test
-            | where qstr("last_name: Smith") or emp_no > 10010
+            | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32")
+            """;
+        var analyzer = makeAnalyzer("mapping-all-types.json");
+        var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer);
+
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var field = as(project.child(), FieldExtractExec.class);
+        var query = as(field.child(), EsQueryExec.class);
+        assertThat(query.limit().fold(), is(1000));
+
+        Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")");
+        var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource);
+        var queryString = QueryBuilders.queryStringQuery("last_name: Smith");
+        var expected = QueryBuilders.boolQuery().must(queryString).must(terms);
+        assertThat(query.query().toString(), is(expected.toString()));
+    }
+
+    /**
+     * Expecting
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua
+     * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162],false]
+     *   \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua
+     * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]]
+     *     \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#]
+     *       \_EsQueryExec[test], indexMode[standard],
+     *       query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},
+     *       {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}],
+     *       "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324]
+     */
+    public void testQueryStringFunctionMultipleWhereClauses() {
+        String queryText = """
+            from test
+            | where qstr("last_name: Smith")
+            | where emp_no > 10010
             """;
         var plan = plannerOptimizer.plan(queryText, IS_SV_STATS);
 
@@ -462,34 +499,31 @@ public void testQueryStringFunctionDisjunctionWhereClauses() {
         var query = as(field.child(), EsQueryExec.class);
         assertThat(query.limit().fold(), is(1000));
 
-        Source filterSource = new Source(2, 36, "emp_no > 10000");
+        Source filterSource = new Source(3, 8, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
         var queryString = QueryBuilders.queryStringQuery("last_name: Smith");
-        var expected = QueryBuilders.boolQuery().should(queryString).should(range);
+        var expected = QueryBuilders.boolQuery().must(queryString).must(range);
         assertThat(query.query().toString(), is(expected.toString()));
     }
 
     /**
      * Expecting
      * LimitExec[1000[INTEGER]]
-     * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_
-     * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16],
-     * false]
-     *   \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_
-     * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16]
-     *     \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..]
-     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{
-     *       "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}},
-     *       "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354]
+     * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
+     * me{f}#6, long_noidx{f}#11, salary{f}#7],false]
+     *   \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
+     * me{f}#6, long_noidx{f}#11, salary{f}#7]]
+     *     \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":
+     *       {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},
+     *       {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}]
      */
-    public void testQueryStringFunctionWithFunctionsPushedToLucene() {
-        assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+    public void testQueryStringFunctionMultipleQstrClauses() {
         String queryText = """
             from test
-            | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32")
+            | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]")
             """;
-        var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution());
-        var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer);
+        var plan = plannerOptimizer.plan(queryText, IS_SV_STATS);
 
         var limit = as(plan, LimitExec.class);
         var exchange = as(limit.child(), ExchangeExec.class);
@@ -498,45 +532,103 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() {
         var query = as(field.child(), EsQueryExec.class);
         assertThat(query.limit().fold(), is(1000));
 
-        Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")");
-        var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource);
-        var queryString = QueryBuilders.queryStringQuery("last_name: Smith");
-        var expected = QueryBuilders.boolQuery().must(queryString).must(terms);
+        var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith");
+        var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]");
+        var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight);
         assertThat(query.query().toString(), is(expected.toString()));
     }
 
     /**
      * Expecting
-     *LimitExec[1000[INTEGER]]
-     * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n
-     * ame{f}#7, long_noidx{f}#12, salary{f}#8],false]
-     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n
-     * ame{f}#7, long_noidx{f}#12, salary{f}#8]]
-     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, gender{f}#5, job{f}#]
-     *       \_LimitExec[1000[INTEGER]]
-     *         \_FilterExec[LENGTH(first_name{f}#4) > 10[INTEGER]]
-     *           \_FieldExtractExec[first_name{f}#4]
-     *             \_EsQueryExec[test], indexMode[standard],
-     *             query[{"query_string":{"query":"last_name: Smith","fields":[]}}][_doc{f}#13], limit[], sort[] estimatedRowSize[324]
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
+     * me{f}#6, long_noidx{f}#11, salary{f}#7],false]
+     *   \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
+     * me{f}#6, long_noidx{f}#11, salary{f}#7]]
+     *     \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"match":{"last_name":{"query":"Smith"}}}]
+     */
+    public void testMatchFunction() {
+        var plan = plannerOptimizer.plan("""
+            from test
+            | where match(last_name, "Smith")
+            """, IS_SV_STATS);
+
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var field = as(project.child(), FieldExtractExec.class);
+        var query = as(field.child(), EsQueryExec.class);
+        assertThat(query.limit().fold(), is(1000));
+        var expected = QueryBuilders.matchQuery("last_name", "Smith");
+        assertThat(query.query().toString(), is(expected.toString()));
+    }
+
+    /**
+     * Expecting
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua
+     * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418],false]
+     *   \_ProjectExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua
+     * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418]]
+     *     \_FieldExtractExec[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#]
+     *       \EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}},
+     *       {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},
+     *       "source":"emp_no > 10010@2:39"}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324]
      */
-    public void testQueryStringFunctionWithFunctionNotPushedDown() {
-        assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+    public void testMatchFunctionConjunctionWhereOperands() {
         String queryText = """
             from test
-            | where qstr("last_name: Smith") and length(first_name) > 10
+            | where match(last_name, "Smith") and emp_no > 10010
             """;
         var plan = plannerOptimizer.plan(queryText, IS_SV_STATS);
 
-        var firstLimit = as(plan, LimitExec.class);
-        var exchange = as(firstLimit.child(), ExchangeExec.class);
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
-        var secondLimit = as(field.child(), LimitExec.class);
-        var filter = as(secondLimit.child(), FilterExec.class);
-        var fieldExtract = as(filter.child(), FieldExtractExec.class);
-        var query = as(fieldExtract.child(), EsQueryExec.class);
+        var query = as(field.child(), EsQueryExec.class);
+        assertThat(query.limit().fold(), is(1000));
 
-        var expected = QueryBuilders.queryStringQuery("last_name: Smith");
+        Source filterSource = new Source(2, 38, "emp_no > 10000");
+        var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
+        var queryString = QueryBuilders.matchQuery("last_name", "Smith");
+        var expected = QueryBuilders.boolQuery().must(queryString).must(range);
+        assertThat(query.query().toString(), is(expected.toString()));
+    }
+
+    /**
+     * Expecting
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_
+     * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16],
+     * false]
+     *   \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_
+     * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16]
+     *     \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"text":{"query":"beta"}}},
+     *       {"esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}},
+     *       "source":"cidr_match(ip, \"127.0.0.1/32\")@2:33"}}],"boost":1.0}}][_doc{f}#22], limit[1000], sort[] estimatedRowSize[354]
+     */
+    public void testMatchFunctionWithFunctionsPushedToLucene() {
+        String queryText = """
+            from test
+            | where match(text, "beta") and cidr_match(ip, "127.0.0.1/32")
+            """;
+        var analyzer = makeAnalyzer("mapping-all-types.json");
+        var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer);
+
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var field = as(project.child(), FieldExtractExec.class);
+        var query = as(field.child(), EsQueryExec.class);
+        assertThat(query.limit().fold(), is(1000));
+
+        Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")");
+        var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource);
+        var queryString = QueryBuilders.matchQuery("text", "beta");
+        var expected = QueryBuilders.boolQuery().must(queryString).must(terms);
         assertThat(query.query().toString(), is(expected.toString()));
     }
 
@@ -548,16 +640,14 @@ public void testQueryStringFunctionWithFunctionNotPushedDown() {
      *   \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua
      * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]]
      *     \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#]
-     *       \_EsQueryExec[test], indexMode[standard],
-     *       query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},
-     *       {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}],
-     *       "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}},
+     *       {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},
+     *       "source":"emp_no > 10010@3:9"}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324]
      */
-    public void testQueryStringFunctionMultipleWhereClauses() {
-        assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+    public void testMatchFunctionMultipleWhereClauses() {
         String queryText = """
             from test
-            | where qstr("last_name: Smith")
+            | where match(last_name, "Smith")
             | where emp_no > 10010
             """;
         var plan = plannerOptimizer.plan(queryText, IS_SV_STATS);
@@ -571,7 +661,7 @@ public void testQueryStringFunctionMultipleWhereClauses() {
 
         Source filterSource = new Source(3, 8, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
-        var queryString = QueryBuilders.queryStringQuery("last_name: Smith");
+        var queryString = QueryBuilders.matchQuery("last_name", "Smith");
         var expected = QueryBuilders.boolQuery().must(queryString).must(range);
         assertThat(query.query().toString(), is(expected.toString()));
     }
@@ -584,15 +674,13 @@ public void testQueryStringFunctionMultipleWhereClauses() {
      *   \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
      * me{f}#6, long_noidx{f}#11, salary{f}#7]]
      *     \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen]
-     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":
-     *       {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},
-     *       {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}},
+     *       {"match":{"first_name":{"query":"John"}}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324]
      */
-    public void testQueryStringFunctionMultipleQstrClauses() {
-        assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
+    public void testMatchFunctionMultipleQstrClauses() {
         String queryText = """
             from test
-            | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]")
+            | where match(last_name, "Smith") and match(first_name, "John")
             """;
         var plan = plannerOptimizer.plan(queryText, IS_SV_STATS);
 
@@ -603,8 +691,8 @@ public void testQueryStringFunctionMultipleQstrClauses() {
         var query = as(field.child(), EsQueryExec.class);
         assertThat(query.limit().fold(), is(1000));
 
-        var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith");
-        var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]");
+        var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith");
+        var queryStringRight = QueryBuilders.matchQuery("first_name", "John");
         var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight);
         assertThat(query.query().toString(), is(expected.toString()));
     }
@@ -809,8 +897,15 @@ public void testIsNotNull_TextField_Pushdown_WithCount() {
 
     private record OutOfRangeTestCase(String fieldName, String tooLow, String tooHigh) {};
 
+    private static final String LT = "<";
+    private static final String LTE = "<=";
+    private static final String GT = ">";
+    private static final String GTE = ">=";
+    private static final String EQ = "==";
+    private static final String NEQ = "!=";
+
     public void testOutOfRangeFilterPushdown() {
-        var allTypeMappingAnalyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution());
+        var allTypeMappingAnalyzer = makeAnalyzer("mapping-all-types.json");
 
         String largerThanInteger = String.valueOf(randomLongBetween(Integer.MAX_VALUE + 1L, Long.MAX_VALUE));
         String smallerThanInteger = String.valueOf(randomLongBetween(Long.MIN_VALUE, Integer.MIN_VALUE - 1L));
@@ -827,16 +922,8 @@ public void testOutOfRangeFilterPushdown() {
             new OutOfRangeTestCase("integer", smallerThanInteger, largerThanInteger),
             new OutOfRangeTestCase("long", smallerThanLong, largerThanLong)
             // TODO: add unsigned_long https://github.com/elastic/elasticsearch/issues/102935
-            // TODO: add half_float, float https://github.com/elastic/elasticsearch/issues/100130
         );
 
-        final String LT = "<";
-        final String LTE = "<=";
-        final String GT = ">";
-        final String GTE = ">=";
-        final String EQ = "==";
-        final String NEQ = "!=";
-
         for (OutOfRangeTestCase testCase : cases) {
             List<String> trueForSingleValuesPredicates = List.of(
                 LT + testCase.tooHigh,
@@ -889,6 +976,51 @@ public void testOutOfRangeFilterPushdown() {
         }
     }
 
+    public void testOutOfRangeFilterPushdownWithFloatAndHalfFloat() {
+        var allTypeMappingAnalyzer = makeAnalyzer("mapping-all-types.json");
+
+        String smallerThanFloat = String.valueOf(randomDoubleBetween(-Double.MAX_VALUE, -Float.MAX_VALUE - 1d, true));
+        String largerThanFloat = String.valueOf(randomDoubleBetween(Float.MAX_VALUE + 1d, Double.MAX_VALUE, true));
+
+        List<OutOfRangeTestCase> cases = List.of(
+            new OutOfRangeTestCase("float", smallerThanFloat, largerThanFloat),
+            new OutOfRangeTestCase("half_float", smallerThanFloat, largerThanFloat)
+        );
+
+        for (OutOfRangeTestCase testCase : cases) {
+            for (var value : List.of(testCase.tooHigh, testCase.tooLow)) {
+                for (String predicate : List.of(LT, LTE, GT, GTE, EQ, NEQ)) {
+                    String comparison = testCase.fieldName + predicate + value;
+                    var query = "from test | where " + comparison;
+
+                    Source expectedSource = new Source(1, 18, comparison);
+
+                    logger.info("Query: " + query);
+                    EsQueryExec actualQueryExec = doTestOutOfRangeFilterPushdown(query, allTypeMappingAnalyzer);
+
+                    assertThat(actualQueryExec.query(), is(instanceOf(SingleValueQuery.Builder.class)));
+                    var actualLuceneQuery = (SingleValueQuery.Builder) actualQueryExec.query();
+                    assertThat(actualLuceneQuery.field(), equalTo(testCase.fieldName));
+                    assertThat(actualLuceneQuery.source(), equalTo(expectedSource));
+
+                    QueryBuilder actualInnerLuceneQuery = actualLuceneQuery.next();
+
+                    if (predicate.equals(EQ)) {
+                        QueryBuilder expectedInnerQuery = QueryBuilders.termQuery(testCase.fieldName, Double.parseDouble(value));
+                        assertThat(actualInnerLuceneQuery, equalTo(expectedInnerQuery));
+                    } else if (predicate.equals(NEQ)) {
+                        QueryBuilder expectedInnerQuery = QueryBuilders.boolQuery()
+                            .mustNot(QueryBuilders.termQuery(testCase.fieldName, Double.parseDouble(value)));
+                        assertThat(actualInnerLuceneQuery, equalTo(expectedInnerQuery));
+                    } else { // one of LT, LTE, GT, GTE
+                        assertTrue(actualInnerLuceneQuery instanceof RangeQueryBuilder);
+                        assertThat(((RangeQueryBuilder) actualInnerLuceneQuery).fieldName(), equalTo(testCase.fieldName));
+                    }
+                }
+            }
+        }
+    }
+
     /**
      * Expects e.g.
      * LimitExec[1000[INTEGER]]
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index 6ff541de1854a..ff7675504d6ff 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -537,6 +537,24 @@ public void testCombineProjectionWithDuplicateAggregation() {
         assertThat(Expressions.names(agg.groupings()), contains("last_name", "first_name"));
     }
 
+    /**
+     * Limit[1000[INTEGER]]
+     * \_Aggregate[STANDARD,[],[SUM(salary{f}#12,true[BOOLEAN]) AS sum(salary), SUM(salary{f}#12,last_name{f}#11 == [44 6f 65][KEYW
+     * ORD]) AS sum(salary) WheRe last_name ==   "Doe"]]
+     *   \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
+     */
+    public void testStatsWithFilteringDefaultAliasing() {
+        var plan = plan("""
+            from test
+            | stats sum(salary), sum(salary) WheRe last_name ==   "Doe"
+            """);
+
+        var limit = as(plan, Limit.class);
+        var agg = as(limit.child(), Aggregate.class);
+        assertThat(agg.aggregates(), hasSize(2));
+        assertThat(Expressions.names(agg.aggregates()), contains("sum(salary)", "sum(salary) WheRe last_name ==   \"Doe\""));
+    }
+
     public void testQlComparisonOptimizationsApply() {
         var plan = plan("""
             from test
@@ -720,6 +738,7 @@ public void testMultipleCombineLimits() {
         );
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/115311")
     public void testSelectivelyPushDownFilterPastRefAgg() {
         // expected plan: "from test | where emp_no > 1 and emp_no < 3 | stats x = count(1) by emp_no | where x > 7"
         LogicalPlan plan = optimizedPlan("""
@@ -772,6 +791,7 @@ public void testNoPushDownOrFilterPastAgg() {
         assertTrue(stats.child() instanceof EsRelation);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/115311")
     public void testSelectivePushDownComplexFilterPastAgg() {
         // expected plan: from test | emp_no > 0 | stats x = count(1) by emp_no | where emp_no < 3 or x > 9
         LogicalPlan plan = optimizedPlan("""
@@ -1375,13 +1395,15 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand2() {
     }
 
     /**
+     * TODO: Push down the filter correctly https://github.com/elastic/elasticsearch/issues/115311
+     *
      * Expected
      * Limit[5[INTEGER]]
-     *  \_Aggregate[[first_name{f}#232],[MAX(salary{f}#233) AS max_s, first_name{f}#232]]
-     *    \_Filter[ISNOTNULL(first_name{f}#232)]
-     *      \_MvExpand[first_name{f}#232]
-     *        \_TopN[[Order[emp_no{f}#231,ASC,LAST]],50[INTEGER]]
-     *          \_EsRelation[employees][emp_no{f}#231, first_name{f}#232, salary{f}#233]
+     * \_Filter[ISNOTNULL(first_name{r}#23)]
+     *   \_Aggregate[STANDARD,[first_name{r}#23],[MAX(salary{f}#18,true[BOOLEAN]) AS max_s, first_name{r}#23]]
+     *     \_MvExpand[first_name{f}#14,first_name{r}#23]
+     *       \_TopN[[Order[emp_no{f}#13,ASC,LAST]],50[INTEGER]]
+     *         \_EsRelation[test][_meta_field{f}#19, emp_no{f}#13, first_name{f}#14, ..]
      */
     public void testDontPushDownLimitPastAggregate_AndMvExpand() {
         LogicalPlan plan = optimizedPlan("""
@@ -1395,10 +1417,10 @@ public void testDontPushDownLimitPastAggregate_AndMvExpand() {
             | limit 5""");
 
         var limit = as(plan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
         assertThat(limit.limit().fold(), equalTo(5));
-        var agg = as(limit.child(), Aggregate.class);
-        var filter = as(agg.child(), Filter.class);
-        var mvExp = as(filter.child(), MvExpand.class);
+        var agg = as(filter.child(), Aggregate.class);
+        var mvExp = as(agg.child(), MvExpand.class);
         var topN = as(mvExp.child(), TopN.class);
         assertThat(topN.limit().fold(), equalTo(50));
         assertThat(orderNames(topN), contains("emp_no"));
@@ -1406,14 +1428,16 @@ public void testDontPushDownLimitPastAggregate_AndMvExpand() {
     }
 
     /**
+     * TODO: Push down the filter correctly https://github.com/elastic/elasticsearch/issues/115311
+     *
      * Expected
      * Limit[5[INTEGER]]
-     *  \_Aggregate[[first_name{f}#262],[MAX(salary{f}#263) AS max_s, first_name{f}#262]]
-     *    \_Filter[ISNOTNULL(first_name{f}#262)]
-     *      \_Limit[50[INTEGER]]
-     *        \_MvExpand[first_name{f}#262]
-     *          \_Limit[50[INTEGER]]
-     *            \_EsRelation[employees][emp_no{f}#261, first_name{f}#262, salary{f}#263]
+     * \_Filter[ISNOTNULL(first_name{r}#22)]
+     *   \_Aggregate[STANDARD,[first_name{r}#22],[MAX(salary{f}#17,true[BOOLEAN]) AS max_s, first_name{r}#22]]
+     *     \_Limit[50[INTEGER]]
+     *       \_MvExpand[first_name{f}#13,first_name{r}#22]
+     *         \_Limit[50[INTEGER]]
+     *           \_EsRelation[test][_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, ..]
      */
     public void testPushDown_TheRightLimit_PastMvExpand() {
         LogicalPlan plan = optimizedPlan("""
@@ -1427,9 +1451,9 @@ public void testPushDown_TheRightLimit_PastMvExpand() {
 
         var limit = as(plan, Limit.class);
         assertThat(limit.limit().fold(), equalTo(5));
-        var agg = as(limit.child(), Aggregate.class);
-        var filter = as(agg.child(), Filter.class);
-        limit = as(filter.child(), Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        var agg = as(filter.child(), Aggregate.class);
+        limit = as(agg.child(), Limit.class);
         assertThat(limit.limit().fold(), equalTo(50));
         var mvExp = as(limit.child(), MvExpand.class);
         limit = as(mvExp.child(), Limit.class);
@@ -5565,6 +5589,38 @@ public void testToDatePeriodToTimeDurationWithField() {
         assertEquals("1:60: argument of [to_timeduration(x)] must be a constant, received [x]", e.getMessage().substring(header.length()));
     }
 
+    // These should pass eventually once we lift some restrictions on match function
+    public void testMatchWithNonIndexedColumnCurrentlyUnsupported() {
+        final String header = "Found 1 problem\nline ";
+        VerificationException e = expectThrows(VerificationException.class, () -> plan("""
+            from test | eval initial = substring(first_name, 1) | where match(initial, "A")"""));
+        assertTrue(e.getMessage().startsWith("Found "));
+        assertEquals(
+            "1:67: [MATCH] cannot operate on [initial], which is not a field from an index mapping",
+            e.getMessage().substring(header.length())
+        );
+
+        e = expectThrows(VerificationException.class, () -> plan("""
+            from test | eval text=concat(first_name, last_name) | where match(text, "cat")"""));
+        assertTrue(e.getMessage().startsWith("Found "));
+        assertEquals(
+            "1:67: [MATCH] cannot operate on [text], which is not a field from an index mapping",
+            e.getMessage().substring(header.length())
+        );
+    }
+
+    public void testMatchFunctionIsNotNullable() {
+        String queryText = """
+            row n = null | eval text = n + 5 | where match(text::keyword, "Anna")
+            """;
+
+        VerificationException ve = expectThrows(VerificationException.class, () -> plan(queryText));
+        assertThat(
+            ve.getMessage(),
+            containsString("[MATCH] cannot operate on [text::keyword], which is not a field from an index mapping")
+        );
+    }
+
     private Literal nullOf(DataType dataType) {
         return new Literal(Source.EMPTY, null, dataType);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
index 6746b8ff61268..961c70acada7b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
@@ -172,7 +172,7 @@
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
-// @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug")
+// @TestLogging(value = "org.elasticsearch.xpack.esql:DEBUG", reason = "debug")
 public class PhysicalPlanOptimizerTests extends ESTestCase {
 
     private static final String PARAM_FORMATTING = "%1$s";
@@ -3211,28 +3211,28 @@ public void testPushSpatialIntersectsStringToSource() {
     /**
      * Plan:
      * <code>
-     * LimitExec[1000[INTEGER]]
-     * \_ExchangeExec[[],false]
-     *   \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[
+     * EvalExec[[scalerank{f}#8 AS rank]]
+     * \_LimitExec[1000[INTEGER]]
+     *   \_ExchangeExec[[],false]
+     *     \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[
      * Limit[1000[INTEGER]]
-     * \_Filter[rank{r}#4 lt 4[INTEGER]]
-     *   \_Eval[[scalerank{f}#8 AS rank]]
-     *     \_EsRelation[airports][abbrev{f}#6, city{f}#12, city_location{f}#13, count..]]]
+     * \_Filter[scalerank{f}#8 &lt; 4[INTEGER]]
+     *   \_EsRelation[airports][abbrev{f}#6, city{f}#12, city_location{f}#13, count..]]]
      * </code>
      * Optimized:
      * <code>
-     * LimitExec[1000[INTEGER]]
-     * \_ExchangeExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8,
-     *     type{f}#9, rank{r}#4],false]
-     *   \_ProjectExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8,
-     *       type{f}#9, rank{r}#4]]
-     *     \_FieldExtractExec[abbrev{f}#6, city{f}#12, city_location{f}#13, count..][]
-     *       \_LimitExec[1000[INTEGER]]
-     *         \_EvalExec[[scalerank{f}#8 AS rank]]
-     *           \_FieldExtractExec[scalerank{f}#8][]
-     *             \_EsQueryExec[airports], indexMode[standard], query[{"
-     *               esql_single_value":{"field":"scalerank","next":{"range":{"scalerank":{"lt":4,"boost":1.0}}},"source":"rank &lt; 4@3:9"}
-     *             }][_doc{f}#23], limit[], sort[] estimatedRowSize[304]
+     * EvalExec[[scalerank{f}#8 AS rank]]
+     * \_LimitExec[1000[INTEGER]]
+     *   \_ExchangeExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8,
+     *       type{f}#9],false
+     *     ]
+     *     \_ProjectExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8,
+     *         type{f}#9]
+     *       ]
+     *       \_FieldExtractExec[abbrev{f}#6, city{f}#12, city_location{f}#13, count..][]
+     *         \_EsQueryExec[airports], indexMode[standard], query[{
+     *           "esql_single_value":{"field":"scalerank","next":{"range":{"scalerank":{"lt":4,"boost":1.0}}},"source":"rank &lt; 4@3:9"}
+     *          ][_doc{f}#23], limit[1000], sort[] estimatedRowSize[304]
      * </code>
      */
     public void testPushWhereEvalToSource() {
@@ -3243,7 +3243,8 @@ public void testPushWhereEvalToSource() {
             """;
 
         var plan = this.physicalPlan(query, airports);
-        var limit = as(plan, LimitExec.class);
+        var eval = as(plan, EvalExec.class);
+        var limit = as(eval.child(), LimitExec.class);
         var exchange = as(limit.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
         var limit2 = as(fragment.fragment(), Limit.class);
@@ -3251,16 +3252,14 @@ public void testPushWhereEvalToSource() {
         assertThat("filter contains LessThan", filter.condition(), instanceOf(LessThan.class));
 
         var optimized = optimizedPlan(plan);
-        var topLimit = as(optimized, LimitExec.class);
+        eval = as(optimized, EvalExec.class);
+        var topLimit = as(eval.child(), LimitExec.class);
         exchange = as(topLimit.child(), ExchangeExec.class);
         var project = as(exchange.child(), ProjectExec.class);
         var fieldExtract = as(project.child(), FieldExtractExec.class);
         assertThat(fieldExtract.attributesToExtract().size(), greaterThan(5));
-        limit = as(fieldExtract.child(), LimitExec.class);
-        var eval = as(limit.child(), EvalExec.class);
-        fieldExtract = as(eval.child(), FieldExtractExec.class);
-        assertThat(fieldExtract.attributesToExtract().stream().map(Attribute::name).collect(Collectors.toList()), contains("scalerank"));
         var source = source(fieldExtract.child());
+        assertThat(source.limit(), is(topLimit.limit()));
         var condition = as(source.query(), SingleValueQuery.Builder.class);
         assertThat("Expected predicate to be passed to Lucene query", condition.source().text(), equalTo("rank < 4"));
         assertThat("Expected field to be passed to Lucene query", condition.field(), equalTo("scalerank"));
@@ -3281,7 +3280,8 @@ public void testPushSpatialIntersectsEvalToSource() {
             """ }) {
 
             var plan = this.physicalPlan(query, airports);
-            var limit = as(plan, LimitExec.class);
+            var eval = as(plan, EvalExec.class);
+            var limit = as(eval.child(), LimitExec.class);
             var exchange = as(limit.child(), ExchangeExec.class);
             var fragment = as(exchange.child(), FragmentExec.class);
             var limit2 = as(fragment.fragment(), Limit.class);
@@ -3289,16 +3289,14 @@ public void testPushSpatialIntersectsEvalToSource() {
             assertThat("filter contains ST_INTERSECTS", filter.condition(), instanceOf(SpatialIntersects.class));
 
             var optimized = optimizedPlan(plan);
-            var topLimit = as(optimized, LimitExec.class);
+            eval = as(optimized, EvalExec.class);
+            var topLimit = as(eval.child(), LimitExec.class);
             exchange = as(topLimit.child(), ExchangeExec.class);
             var project = as(exchange.child(), ProjectExec.class);
             var fieldExtract = as(project.child(), FieldExtractExec.class);
             assertThat(fieldExtract.attributesToExtract().size(), greaterThan(5));
-            limit = as(fieldExtract.child(), LimitExec.class);
-            var eval = as(limit.child(), EvalExec.class);
-            fieldExtract = as(eval.child(), FieldExtractExec.class);
-            assertThat(fieldExtract.attributesToExtract().stream().map(Attribute::name).collect(Collectors.toList()), contains("location"));
             var source = source(fieldExtract.child());
+            assertThat(source.limit(), is(topLimit.limit()));
             var condition = as(source.query(), SpatialRelatesQuery.ShapeQueryBuilder.class);
             assertThat("Geometry field name", condition.fieldName(), equalTo("location"));
             assertThat("Spatial relationship", condition.relation(), equalTo(ShapeRelation.INTERSECTS));
@@ -4771,12 +4769,24 @@ public void testPushTopNInlineDistanceToSource() {
         var exchange = asRemoteExchange(topN.child());
 
         project = as(exchange.child(), ProjectExec.class);
-        assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "$$order_by$0$0"));
+        // Depending on what is run before this test, the synthetic name could have variable suffixes, so we must only assert on the prefix
+        assertThat(
+            names(project.projections()),
+            contains(
+                equalTo("abbrev"),
+                equalTo("name"),
+                equalTo("location"),
+                equalTo("country"),
+                equalTo("city"),
+                startsWith("$$order_by$0$")
+            )
+        );
         var extract = as(project.child(), FieldExtractExec.class);
         assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city"));
         var evalExec = as(extract.child(), EvalExec.class);
         var alias = as(evalExec.fields().get(0), Alias.class);
-        assertThat(alias.name(), is("$$order_by$0$0"));
+        assertThat(alias.name(), startsWith("$$order_by$0$"));
+        var aliasName = alias.name();  // We need this name to know what to assert on later when comparing the Order to the Sort
         var stDistance = as(alias.child(), StDistance.class);
         assertThat(stDistance.left().toString(), startsWith("location"));
         extract = as(evalExec.child(), FieldExtractExec.class);
@@ -4786,7 +4796,7 @@ public void testPushTopNInlineDistanceToSource() {
         // Assert that the TopN(distance) is pushed down as geo-sort(location)
         assertThat(source.limit(), is(topN.limit()));
         Set<String> orderSet = orderAsSet(topN.order());
-        Set<String> sortsSet = sortsAsSet(source.sorts(), Map.of("location", "$$order_by$0$0"));
+        Set<String> sortsSet = sortsAsSet(source.sorts(), Map.of("location", aliasName));
         assertThat(orderSet, is(sortsSet));
 
         // Fine-grained checks on the pushed down sort
@@ -5841,14 +5851,14 @@ public void testEnrichBeforeLimit() {
                 | EVAL employee_id = to_str(emp_no)
                 | ENRICH _remote:departments
                 | LIMIT 10""");
-            var enrich = as(plan, EnrichExec.class);
-            assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE));
-            assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2")));
-            var eval = as(enrich.child(), EvalExec.class);
-            var finalLimit = as(eval.child(), LimitExec.class);
+            var finalLimit = as(plan, LimitExec.class);
             var exchange = as(finalLimit.child(), ExchangeExec.class);
             var fragment = as(exchange.child(), FragmentExec.class);
-            var partialLimit = as(fragment.fragment(), Limit.class);
+            var enrich = as(fragment.fragment(), Enrich.class);
+            assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE));
+            assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2")));
+            var evalFragment = as(enrich.child(), Eval.class);
+            var partialLimit = as(evalFragment.child(), Limit.class);
             as(partialLimit.child(), EsRelation.class);
         }
     }
@@ -5891,13 +5901,21 @@ public void testLimitThenEnrich() {
     }
 
     public void testLimitThenEnrichRemote() {
-        var error = expectThrows(VerificationException.class, () -> physicalPlan("""
+        var plan = physicalPlan("""
             FROM test
             | LIMIT 10
             | EVAL employee_id = to_str(emp_no)
             | ENRICH _remote:departments
-            """));
-        assertThat(error.getMessage(), containsString("line 4:3: ENRICH with remote policy can't be executed after LIMIT"));
+            """);
+        var finalLimit = as(plan, LimitExec.class);
+        var exchange = as(finalLimit.child(), ExchangeExec.class);
+        var fragment = as(exchange.child(), FragmentExec.class);
+        var enrich = as(fragment.fragment(), Enrich.class);
+        assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE));
+        assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2")));
+        var evalFragment = as(enrich.child(), Eval.class);
+        var partialLimit = as(evalFragment.child(), Limit.class);
+        as(partialLimit.child(), EsRelation.class);
     }
 
     public void testEnrichBeforeTopN() {
@@ -5951,6 +5969,23 @@ public void testEnrichBeforeTopN() {
             var eval = as(enrich.child(), Eval.class);
             as(eval.child(), EsRelation.class);
         }
+        {
+            var plan = physicalPlan("""
+                FROM test
+                | EVAL employee_id = to_str(emp_no)
+                | ENRICH _remote:departments
+                | SORT department
+                | LIMIT 10""");
+            var topN = as(plan, TopNExec.class);
+            var exchange = as(topN.child(), ExchangeExec.class);
+            var fragment = as(exchange.child(), FragmentExec.class);
+            var partialTopN = as(fragment.fragment(), TopN.class);
+            var enrich = as(partialTopN.child(), Enrich.class);
+            assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE));
+            assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2")));
+            var eval = as(enrich.child(), Eval.class);
+            as(eval.child(), EsRelation.class);
+        }
     }
 
     public void testEnrichAfterTopN() {
@@ -5990,6 +6025,24 @@ public void testEnrichAfterTopN() {
             var partialTopN = as(fragment.fragment(), TopN.class);
             as(partialTopN.child(), EsRelation.class);
         }
+        {
+            var plan = physicalPlan("""
+                FROM test
+                | SORT emp_no
+                | LIMIT 10
+                | EVAL employee_id = to_str(emp_no)
+                | ENRICH _remote:departments
+                """);
+            var topN = as(plan, TopNExec.class);
+            var exchange = as(topN.child(), ExchangeExec.class);
+            var fragment = as(exchange.child(), FragmentExec.class);
+            var enrich = as(fragment.fragment(), Enrich.class);
+            assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE));
+            assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2")));
+            var evalFragment = as(enrich.child(), Eval.class);
+            var partialTopN = as(evalFragment.child(), TopN.class);
+            as(partialTopN.child(), EsRelation.class);
+        }
     }
 
     public void testManyEnrich() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java
index a74ceb4e1426c..c2e85cc43284a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java
@@ -17,8 +17,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
@@ -101,7 +99,6 @@ public void testConstantNot() {
     }
 
     public void testConstantFoldingLikes() {
-        assertEquals(TRUE, new ConstantFolding().rule(new Like(EMPTY, of("test_emp"), new LikePattern("test%", (char) 0))).canonical());
         assertEquals(TRUE, new ConstantFolding().rule(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical());
         assertEquals(TRUE, new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java
index 49a738f4f4fa3..e159e5ed0bd7d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java
@@ -213,6 +213,7 @@ public void testPushDownLikeRlikeFilter() {
 
     // from ... | where a > 1 | stats count(1) by b | where count(1) >= 3 and b < 2
     // => ... | where a > 1 and b < 2 | stats count(1) by b | where count(1) >= 3
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/115311")
     public void testSelectivelyPushDownFilterPastFunctionAgg() {
         EsRelation relation = relation();
         GreaterThan conditionA = greaterThanOf(getFieldAttribute("a"), ONE);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java
index c5e64d41be4dc..20d638a113bf2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
@@ -26,18 +24,6 @@
 
 public class ReplaceRegexMatchTests extends ESTestCase {
 
-    public void testMatchAllLikeToExist() {
-        for (String s : asList("%", "%%", "%%%")) {
-            LikePattern pattern = new LikePattern(s, (char) 0);
-            FieldAttribute fa = getFieldAttribute();
-            Like l = new Like(EMPTY, fa, pattern);
-            Expression e = new ReplaceRegexMatch().rule(l);
-            assertEquals(IsNotNull.class, e.getClass());
-            IsNotNull inn = (IsNotNull) e;
-            assertEquals(fa, inn.field());
-        }
-    }
-
     public void testMatchAllWildcardLikeToExist() {
         for (String s : asList("*", "**", "***")) {
             WildcardPattern pattern = new WildcardPattern(s);
@@ -60,31 +46,19 @@ public void testMatchAllRLikeToExist() {
         assertEquals(fa, inn.field());
     }
 
-    public void testExactMatchLike() {
-        for (String s : asList("ab", "ab0%", "ab0_c")) {
-            LikePattern pattern = new LikePattern(s, '0');
+    public void testExactMatchWildcardLike() {
+        for (String s : asList("ab", "ab\\*", "ab\\?c")) {
+            WildcardPattern pattern = new WildcardPattern(s);
             FieldAttribute fa = getFieldAttribute();
-            Like l = new Like(EMPTY, fa, pattern);
+            WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
             Expression e = new ReplaceRegexMatch().rule(l);
             assertEquals(Equals.class, e.getClass());
             Equals eq = (Equals) e;
             assertEquals(fa, eq.left());
-            assertEquals(s.replace("0", StringUtils.EMPTY), eq.right().fold());
+            assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold());
         }
     }
 
-    public void testExactMatchWildcardLike() {
-        String s = "ab";
-        WildcardPattern pattern = new WildcardPattern(s);
-        FieldAttribute fa = getFieldAttribute();
-        WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
-        Expression e = new ReplaceRegexMatch().rule(l);
-        assertEquals(Equals.class, e.getClass());
-        Equals eq = (Equals) e;
-        assertEquals(fa, eq.left());
-        assertEquals(s, eq.right().fold());
-    }
-
     public void testExactMatchRLike() {
         RLikePattern pattern = new RLikePattern("abc");
         FieldAttribute fa = getFieldAttribute();
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
index a1bcdec2b7c5c..e6fef186721a0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
@@ -144,8 +144,15 @@ void expectError(String query, List<QueryParam> params, String errorMessage) {
         assertThat(e.getMessage(), containsString(errorMessage));
     }
 
-    void expectInvalidIndexNameErrorWithLineNumber(String query, String arg, String lineNumber, String indexString) {
-        expectError(LoggerMessageFormat.format(null, query, arg), lineNumber + "Invalid index name [" + indexString);
+    void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber) {
+        if ((indexString.contains("|") || indexString.contains(" ")) == false) {
+            expectInvalidIndexNameErrorWithLineNumber(query, indexString, lineNumber, indexString);
+        }
+        expectInvalidIndexNameErrorWithLineNumber(query, "\"" + indexString + "\"", lineNumber, indexString);
+    }
+
+    void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String error) {
+        expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + error);
     }
 
     void expectDateMathErrorWithLineNumber(String query, String arg, String lineNumber, String error) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
index 80a2d49d0d94a..67b4dd71260aa 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
@@ -208,7 +208,7 @@ public void testParenthesizedExpression() {
     }
 
     public void testCommandNamesAsIdentifiers() {
-        Expression expr = whereExpression("from and where");
+        Expression expr = whereExpression("from and limit");
         assertThat(expr, instanceOf(And.class));
         And and = (And) expr;
 
@@ -216,7 +216,7 @@ public void testCommandNamesAsIdentifiers() {
         assertThat(((UnresolvedAttribute) and.left()).name(), equalTo("from"));
 
         assertThat(and.right(), instanceOf(UnresolvedAttribute.class));
-        assertThat(((UnresolvedAttribute) and.right()).name(), equalTo("where"));
+        assertThat(((UnresolvedAttribute) and.right()).name(), equalTo("limit"));
     }
 
     public void testIdentifiersCaseSensitive() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
index 53621a79aedac..8019dbf77ffbf 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -20,14 +21,18 @@
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
 import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
@@ -54,6 +59,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -321,6 +327,61 @@ public void testAggsWithGroupKeyAsAgg() throws Exception {
         }
     }
 
+    public void testStatsWithGroupKeyAndAggFilter() throws Exception {
+        var a = attribute("a");
+        var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a));
+        var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1))));
+        assertEquals(
+            new Aggregate(EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, List.of(a), List.of(filter, a)),
+            processingCommand("stats min(a) where a > 1 by a")
+        );
+    }
+
+    public void testStatsWithGroupKeyAndMixedAggAndFilter() throws Exception {
+        var a = attribute("a");
+        var min = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a));
+        var max = new UnresolvedFunction(EMPTY, "max", DEFAULT, List.of(a));
+        var avg = new UnresolvedFunction(EMPTY, "avg", DEFAULT, List.of(a));
+        var min_alias = new Alias(EMPTY, "min", min);
+
+        var max_filter_ex = new Or(
+            EMPTY,
+            new GreaterThan(EMPTY, new Mod(EMPTY, a, integer(3)), integer(10)),
+            new GreaterThan(EMPTY, new Div(EMPTY, a, integer(2)), integer(100))
+        );
+        var max_filter = new Alias(EMPTY, "max", new FilteredExpression(EMPTY, max, max_filter_ex));
+
+        var avg_filter_ex = new GreaterThan(EMPTY, new Div(EMPTY, a, integer(2)), integer(100));
+        var avg_filter = new Alias(EMPTY, "avg", new FilteredExpression(EMPTY, avg, avg_filter_ex));
+
+        assertEquals(
+            new Aggregate(
+                EMPTY,
+                PROCESSING_CMD_INPUT,
+                Aggregate.AggregateType.STANDARD,
+                List.of(a),
+                List.of(min_alias, max_filter, avg_filter, a)
+            ),
+            processingCommand("""
+                stats
+                min = min(a),
+                max = max(a) WHERE (a % 3 > 10 OR a / 2 > 100),
+                avg = avg(a) WHERE a / 2 > 100
+                BY a
+                """)
+        );
+    }
+
+    public void testStatsWithoutGroupKeyMixedAggAndFilter() throws Exception {
+        var a = attribute("a");
+        var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a));
+        var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1))));
+        assertEquals(
+            new Aggregate(EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, List.of(), List.of(filter)),
+            processingCommand("stats min(a) where a > 1")
+        );
+    }
+
     public void testInlineStatsWithGroups() {
         var query = "inlinestats b = min(a) by c, d.e";
         if (Build.current().isSnapshot() == false) {
@@ -365,7 +426,12 @@ public void testInlineStatsWithoutGroups() {
     }
 
     public void testStringAsIndexPattern() {
-        for (String command : List.of("FROM", "METRICS")) {
+        List<String> commands = new ArrayList<>();
+        commands.add("FROM");
+        if (Build.current().isSnapshot()) {
+            commands.add("METRICS");
+        }
+        for (String command : commands) {
             assertStringAsIndexPattern("foo", command + " \"foo\"");
             assertStringAsIndexPattern("foo,test-*", command + """
                  "foo","test-*"
@@ -397,15 +463,23 @@ public void testStringAsIndexPattern() {
             assertStringAsIndexPattern("`backtick`,``multiple`back``ticks```", command + " `backtick`, ``multiple`back``ticks```");
             assertStringAsIndexPattern("test,metadata,metaata,.metadata", command + " test,\"metadata\", metaata, .metadata");
             assertStringAsIndexPattern(".dot", command + " .dot");
-            assertStringAsIndexPattern("cluster:index", command + " cluster:index");
-            assertStringAsIndexPattern("cluster:.index", command + " cluster:.index");
-            assertStringAsIndexPattern("cluster*:index*", command + " cluster*:index*");
-            assertStringAsIndexPattern("cluster*:*", command + " cluster*:*");
-            assertStringAsIndexPattern("*:index*", command + " *:index*");
-            assertStringAsIndexPattern("*:*", command + " *:*");
+            assertStringAsIndexPattern("cluster:index|pattern", command + " cluster:\"index|pattern\"");
+            assertStringAsIndexPattern("*:index|pattern", command + " \"*:index|pattern\"");
+            clusterAndIndexAsIndexPattern(command, "cluster:index");
+            clusterAndIndexAsIndexPattern(command, "cluster:.index");
+            clusterAndIndexAsIndexPattern(command, "cluster*:index*");
+            clusterAndIndexAsIndexPattern(command, "cluster*:<logstash-{now/D}>*");
+            clusterAndIndexAsIndexPattern(command, "cluster*:*");
+            clusterAndIndexAsIndexPattern(command, "*:index*");
+            clusterAndIndexAsIndexPattern(command, "*:*");
         }
     }
 
+    private void clusterAndIndexAsIndexPattern(String command, String clusterAndIndex) {
+        assertStringAsIndexPattern(clusterAndIndex, command + " " + clusterAndIndex);
+        assertStringAsIndexPattern(clusterAndIndex, command + " \"" + clusterAndIndex + "\"");
+    }
+
     public void testStringAsLookupIndexPattern() {
         assumeTrue("requires snapshot build", Build.current().isSnapshot());
         assertStringAsLookupIndexPattern("foo", "ROW x = 1 | LOOKUP \"foo\" ON j");
@@ -427,221 +501,104 @@ public void testStringAsLookupIndexPattern() {
         assertStringAsLookupIndexPattern("`backtick`", "ROW x = 1 | LOOKUP `backtick` ON j");
         assertStringAsLookupIndexPattern("``multiple`back``ticks```", "ROW x = 1 | LOOKUP ``multiple`back``ticks``` ON j");
         assertStringAsLookupIndexPattern(".dot", "ROW x = 1 | LOOKUP .dot ON j");
-        assertStringAsLookupIndexPattern("cluster:index", "ROW x = 1 | LOOKUP cluster:index ON j");
-        assertStringAsLookupIndexPattern("cluster:.index", "ROW x = 1 | LOOKUP cluster:.index ON j");
-        assertStringAsLookupIndexPattern("cluster*:index*", "ROW x = 1 | LOOKUP cluster*:index* ON j");
-        assertStringAsLookupIndexPattern("cluster*:*", "ROW x = 1 | LOOKUP cluster*:* ON j");
-        assertStringAsLookupIndexPattern("*:index*", "ROW x = 1 | LOOKUP  *:index* ON j");
-        assertStringAsLookupIndexPattern("*:*", "ROW x = 1 | LOOKUP  *:* ON j");
+        clusterAndIndexAsLookupIndexPattern("cluster:index");
+        clusterAndIndexAsLookupIndexPattern("cluster:.index");
+        clusterAndIndexAsLookupIndexPattern("cluster*:index*");
+        clusterAndIndexAsLookupIndexPattern("cluster*:*");
+        clusterAndIndexAsLookupIndexPattern("*:index*");
+        clusterAndIndexAsLookupIndexPattern("*:*");
+    }
+
+    private void clusterAndIndexAsLookupIndexPattern(String clusterAndIndex) {
+        assertStringAsLookupIndexPattern(clusterAndIndex, "ROW x = 1 | LOOKUP " + clusterAndIndex + " ON j");
+        assertStringAsLookupIndexPattern(clusterAndIndex, "ROW x = 1 | LOOKUP \"" + clusterAndIndex + "\"" + " ON j");
     }
 
     public void testInvalidCharacterInIndexPattern() {
         Map<String, String> commands = new HashMap<>();
-        commands.put("FROM {}", "line 1:8: ");
+        commands.put("FROM {}", "line 1:7: ");
         if (Build.current().isSnapshot()) {
-            commands.put("METRICS {}", "line 1:11: ");
-            commands.put("ROW x = 1 | LOOKUP {} ON j", "line 1:22: ");
+            commands.put("METRICS {}", "line 1:10: ");
+            commands.put("ROW x = 1 | LOOKUP {} ON j", "line 1:21: ");
         }
-        List<String> clusterStrings = List.of(" ", " *:", " cluster:");
         String lineNumber;
         for (String command : commands.keySet()) {
             lineNumber = commands.get(command);
-            for (String clusterString : clusterStrings) {
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index|pattern\"", lineNumber, "index|pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index pattern\"", lineNumber, "index pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index#pattern\"", lineNumber, "index#pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index#pattern", lineNumber, "index#pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index?pattern\"", lineNumber, "index?pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index?pattern", lineNumber, "index?pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index>pattern\"", lineNumber, "index>pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index>pattern", lineNumber, "index>pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index<pattern\"", lineNumber, "index<pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index<pattern", lineNumber, "index<pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index/pattern\"", lineNumber, "index/pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index/pattern", lineNumber, "index/pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index\\\\pattern\"", lineNumber, "index\\pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index\\pattern", lineNumber, "index\\pattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"..\"", lineNumber, "..");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "..", lineNumber, "..");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"_indexpattern\"", lineNumber, "_indexpattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "_indexpattern", lineNumber, "_indexpattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"+indexpattern\"", lineNumber, "+indexpattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "+indexpattern", lineNumber, "+indexpattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"--indexpattern\"", lineNumber, "-indexpattern");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "--indexpattern", lineNumber, "-indexpattern");
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "\"<--logstash-{now/M{yyyy.MM}}>\"",
-                    lineNumber,
-                    "-logstash-"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "--<logstash-{now/M{yyyy.MM}}>",
-                    lineNumber,
-                    "-<logstash-{now/M{yyyy.MM}}>"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"<logstash#{now/d}>\"", lineNumber, "logstash#");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "<logstash#{now/d}>", lineNumber, "logstash#");
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "\"+<logstash-{now/d}>\"",
-                    lineNumber,
-                    "+<logstash-{now/d}>"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "+<logstash-{now/d}>",
-                    lineNumber,
-                    "+<logstash-{now/d}>"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "\"_<logstash-{now/d}>\"",
-                    lineNumber,
-                    "_<logstash-{now/d}>"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "_<logstash-{now/d}>",
-                    lineNumber,
-                    "_<logstash-{now/d}>"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"<<logstash{now/d}>>\"", lineNumber, "<logstash");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "<<logstash{now/d}>>", lineNumber, "<logstash");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"<<logstash<{now/d}>>>\"", lineNumber, "<logstash<");
-                expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "<<logstash<{now/d}>>>", lineNumber, "<logstash<");
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "\"-<logstash- {now/d{yyyy.MM.dd|+12:00}}>\"",
-                    lineNumber,
-                    "logstash- "
-                );
-            }
+            expectInvalidIndexNameErrorWithLineNumber(command, "index|pattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "index pattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "index#pattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "index?pattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "index>pattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "index<pattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "index/pattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "_indexpattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "+indexpattern", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "..", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "+<logstash-{now/d}>", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "_<logstash-{now/d}>", lineNumber);
+            expectInvalidIndexNameErrorWithLineNumber(command, "index\\pattern", lineNumber, "index\\pattern");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"index\\\\pattern\"", lineNumber, "index\\pattern");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"--indexpattern\"", lineNumber, "-indexpattern");
+            expectInvalidIndexNameErrorWithLineNumber(command, "--indexpattern", lineNumber, "-indexpattern");
+            expectInvalidIndexNameErrorWithLineNumber(command, "<--logstash-{now/M{yyyy.MM}}>", lineNumber, "-logstash-");
+            expectInvalidIndexNameErrorWithLineNumber(
+                command,
+                "\"--<logstash-{now/M{yyyy.MM}}>\"",
+                lineNumber,
+                "-<logstash-{now/M{yyyy.MM}}>"
+            );
+            expectInvalidIndexNameErrorWithLineNumber(command, "<logstash#{now/d}>", lineNumber, "logstash#");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"<logstash#{now/d}>\"", lineNumber, "logstash#");
+            expectInvalidIndexNameErrorWithLineNumber(command, "<<logstash{now/d}>>", lineNumber, "<logstash");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"<<logstash{now/d}>>\"", lineNumber, "<logstash");
+            expectInvalidIndexNameErrorWithLineNumber(command, "<<logstash<{now/d}>>>", lineNumber, "<logstash<");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"<<logstash<{now/d}>>>\"", lineNumber, "<logstash<");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"-<logstash- {now/d{yyyy.MM.dd|+12:00}}>\"", lineNumber, "logstash- ");
         }
 
-        // comma separated indices
+        // comma separated indices, with exclusions
         // Invalid index names after removing exclusion fail, when there is no index name with wildcard before it
         for (String command : commands.keySet()) {
             if (command.contains("LOOKUP")) {
                 continue;
             }
-            for (String clusterString : clusterStrings) {
-                lineNumber = command.contains("FROM")
-                    ? "line 1:" + (22 + clusterString.length() - 1) + ": "
-                    : "line 1:" + (25 + clusterString.length() - 1) + ": ";
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "indexpattern, --indexpattern",
-                    lineNumber,
-                    "-indexpattern"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "indexpattern, \"--indexpattern\"",
-                    lineNumber,
-                    "-indexpattern"
-                );
-                expectInvalidIndexNameErrorWithLineNumber(
-                    command,
-                    clusterString + "\"indexpattern, --indexpattern\"",
-                    commands.get(command),
-                    "-indexpattern"
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "indexpattern,-indexpattern"),
-                    statement(command, clusterString + "indexpattern, -indexpattern")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "indexpattern,-indexpattern"),
-                    statement(command, clusterString + "indexpattern, \"-indexpattern\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "indexpattern, -indexpattern"),
-                    statement(command, clusterString + "\"indexpattern, -indexpattern\"")
-                );
-            }
+
+            lineNumber = command.contains("FROM") ? "line 1:21: " : "line 1:24: ";
+            expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, --indexpattern", lineNumber, "-indexpattern");
+            expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, \"--indexpattern\"", lineNumber, "-indexpattern");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern, --indexpattern\"", commands.get(command), "-indexpattern");
+            clustersAndIndices(command, "indexpattern", "-indexpattern");
         }
 
         // Invalid index names, except invalid DateMath, are ignored if there is an index name with wildcard before it
+        String dateMathError = "unit [D] not supported for date math [/D]";
         for (String command : commands.keySet()) {
             if (command.contains("LOOKUP")) {
                 continue;
             }
-            for (String clusterString : clusterStrings) {
-                lineNumber = command.contains("FROM")
-                    ? "line 1:" + (11 + clusterString.length() - 1) + ": "
-                    : "line 1:" + (14 + clusterString.length() - 1) + ": ";
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "*,-index#pattern"),
-                    statement(command, clusterString + "*, \"-index#pattern\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "*,-index#pattern"),
-                    statement(command, clusterString + "*, -index#pattern")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "*, -index#pattern"),
-                    statement(command, clusterString + "\"*, -index#pattern\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "index*,-index#pattern"),
-                    statement(command, clusterString + "index*, \"-index#pattern\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "index*,-index#pattern"),
-                    statement(command, clusterString + "index*, -index#pattern")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "index*, -index#pattern"),
-                    statement(command, clusterString + "\"index*, -index#pattern\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "*,-<--logstash-{now/M{yyyy.MM}}>"),
-                    statement(command, clusterString + "*, \"-<--logstash-{now/M{yyyy.MM}}>\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "*,-<--logstash-{now/M{yyyy.MM}}>"),
-                    statement(command, clusterString + "*, -<--logstash-{now/M{yyyy.MM}}>")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "*, -<--logstash-{now/M{yyyy.MM}}>"),
-                    statement(command, clusterString + "\"*, -<--logstash-{now/M{yyyy.MM}}>\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "index*,-<--logstash#-{now/M{yyyy.MM}}>"),
-                    statement(command, clusterString + "index*, \"-<--logstash#-{now/M{yyyy.MM}}>\"")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "index*,-<--logstash#-{now/M{yyyy.MM}}>"),
-                    statement(command, clusterString + "index*, -<--logstash#-{now/M{yyyy.MM}}>")
-                );
-                assertEquals(
-                    unresolvedRelation(clusterString.strip() + "index*, -<--logstash#-{now/M{yyyy.MM}}>"),
-                    statement(command, clusterString + "\"index*, -<--logstash#-{now/M{yyyy.MM}}>\"")
-                );
-                expectDateMathErrorWithLineNumber(
-                    command,
-                    clusterString + "*, \"-<-logstash-{now/D}>\"",
-                    lineNumber,
-                    "unit [D] not supported for date math [/D]"
-                );
-                expectDateMathErrorWithLineNumber(
-                    command,
-                    clusterString + "*, -<-logstash-{now/D}>",
-                    lineNumber,
-                    "unit [D] not supported for date math [/D]"
-                );
-                expectDateMathErrorWithLineNumber(
-                    command,
-                    clusterString + "\"*, -<-logstash-{now/D}>\"",
-                    commands.get(command),
-                    "unit [D] not supported for date math [/D]"
-                );
-            }
+            lineNumber = command.contains("FROM") ? "line 1:10: " : "line 1:13: ";
+            clustersAndIndices(command, "*", "-index#pattern");
+            clustersAndIndices(command, "index*", "-index#pattern");
+            clustersAndIndices(command, "*", "-<--logstash-{now/M{yyyy.MM}}>");
+            clustersAndIndices(command, "index*", "-<--logstash#-{now/M{yyyy.MM}}>");
+            expectDateMathErrorWithLineNumber(command, "*, \"-<-logstash-{now/D}>\"", lineNumber, dateMathError);
+            expectDateMathErrorWithLineNumber(command, "*, -<-logstash-{now/D}>", lineNumber, dateMathError);
+            expectDateMathErrorWithLineNumber(command, "\"*, -<-logstash-{now/D}>\"", commands.get(command), dateMathError);
         }
     }
 
+    private void clustersAndIndices(String command, String indexString1, String indexString2) {
+        assertEquals(unresolvedRelation(indexString1 + "," + indexString2), statement(command, indexString1 + ", " + indexString2));
+        assertEquals(
+            unresolvedRelation(indexString1 + "," + indexString2),
+            statement(command, indexString1 + ", \"" + indexString2 + "\"")
+        );
+        assertEquals(
+            unresolvedRelation(indexString1 + ", " + indexString2),
+            statement(command, "\"" + indexString1 + ", " + indexString2 + "\"")
+        );
+    }
+
     public void testInvalidQuotingAsFromIndexPattern() {
         expectError("FROM \"foo", ": token recognition error at: '\"foo'");
         expectError("FROM \"foo | LIMIT 1", ": token recognition error at: '\"foo | LIMIT 1'");
@@ -925,11 +882,14 @@ public void testDeprecatedIsNullFunction() {
             "row x = is_null(f)",
             "line 1:10: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead"
         );
-        expectError(
-            "from test | eval x = ?fn1(f)",
-            List.of(paramAsIdentifier("fn1", "IS_NULL")),
-            "line 1:23: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead"
-        );
+
+        if (Build.current().isSnapshot()) {
+            expectError(
+                "from test | eval x = ?fn1(f)",
+                List.of(paramAsIdentifier("fn1", "IS_NULL")),
+                "line 1:23: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead"
+            );
+        }
     }
 
     public void testMetadataFieldOnOtherSources() {
@@ -1602,6 +1562,10 @@ public void testIntervalParam() {
     }
 
     public void testParamForIdentifier() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         // field names can appear in eval/where/stats/sort/keep/drop/rename/dissect/grok/enrich/mvexpand
         // eval, where
         assertEquals(
@@ -1859,6 +1823,10 @@ public void testParamForIdentifier() {
     }
 
     public void testParamForIdentifierPattern() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         // name patterns can appear in keep and drop
         // all patterns
         LogicalPlan plan = statement(
@@ -1948,6 +1916,10 @@ public void testParamForIdentifierPattern() {
     }
 
     public void testParamInInvalidPosition() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         // param for pattern is not supported in eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand
         // where/stats/sort/dissect/grok are covered in RestEsqlTestCase
         List<String> invalidParamPositions = List.of("eval ?f1 = 1", "stats x = ?f1(*)", "mv_expand ?f1", "rename ?f1 as ?f2");
@@ -1999,6 +1971,10 @@ public void testParamInInvalidPosition() {
     }
 
     public void testMissingParam() {
+        assumeTrue(
+            "named parameters for identifiers and patterns require snapshot build",
+            EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled()
+        );
         // cover all processing commands eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand/keep/drop
         String error = "Unknown query parameter [f1], did you mean [f4]?";
         String errorMvExpandFunctionNameCommandOption = "Query parameter [?f1] is null or undefined, cannot be used as an identifier";
@@ -2080,13 +2056,11 @@ public void testIdPatternUnquoted() throws Exception {
 
     public void testIdPatternQuoted() throws Exception {
         var string = "`escaped string`";
-        List<String> temp = breakIntoFragments(string);
         assertThat(breakIntoFragments(string), contains(string));
     }
 
     public void testIdPatternQuotedWithDoubleBackticks() throws Exception {
         var string = "`escaped``string`";
-        List<String> temp = breakIntoFragments(string);
         assertThat(breakIntoFragments(string), contains(string));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java
index 1f52795dbacd7..5989c0de6b61d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java
@@ -80,20 +80,67 @@ public void testManyTypeConflicts() throws IOException {
      * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more.
      */
     public void testManyTypeConflictsWithParent() throws IOException {
-        testManyTypeConflicts(true, ByteSizeValue.ofBytes(2774214));
+        testManyTypeConflicts(true, ByteSizeValue.ofBytes(2774192));
         /*
          * History:
          *  2 gb+ - start
          * 43.3mb - Cache attribute subclasses #111447
          *  5.6mb - shorten error messages for UnsupportedAttributes #111973
          *  3.1mb - cache EsFields #112008
-         *  2.6mb - string serialization #112929
+         *  2774214b - string serialization #112929
+         *  2774192b - remove field attribute #112881
          */
     }
 
+    private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) throws IOException {
+        EsIndex index = EsIndexSerializationTests.indexWithManyConflicts(withParent);
+        testSerializePlanWithIndex(index, expected);
+    }
+
+    /**
+     * Test the size of serializing a plan like
+     * FROM index | LIMIT 10
+     * with a single root field that has many children, grandchildren etc.
+     */
+    public void testDeeplyNestedFields() throws IOException {
+        ByteSizeValue expected = ByteSizeValue.ofBytes(47252411);
+        /*
+         * History:
+         *  48223371b - string serialization #112929
+         *  47252411b - remove field attribute #112881
+         */
+
+        int depth = 6;
+        int childrenPerLevel = 8;
+
+        EsIndex index = EsIndexSerializationTests.deeplyNestedIndex(depth, childrenPerLevel);
+        testSerializePlanWithIndex(index, expected);
+    }
+
     /**
-     * Test the size of serializing a plan with many conflicts. Callers of
-     * this method intentionally use a very precise size for the serialized
+     * Test the size of serializing a plan like
+     * FROM index | LIMIT 10 | KEEP one_single_field
+     * with a single root field that has many children, grandchildren etc.
+     */
+    public void testDeeplyNestedFieldsKeepOnlyOne() throws IOException {
+        ByteSizeValue expected = ByteSizeValue.ofBytes(9425806);
+        /*
+         * History:
+         *  9426058b - string serialization #112929
+         *  9425806b - remove field attribute #112881
+         */
+
+        int depth = 6;
+        int childrenPerLevel = 9;
+
+        EsIndex index = EsIndexSerializationTests.deeplyNestedIndex(depth, childrenPerLevel);
+        testSerializePlanWithIndex(index, expected, false);
+    }
+
+    /**
+     * Test the size of serializing the physical plan that will be sent to a data node.
+     * The plan corresponds to `FROM index | LIMIT 10`.
+     * Callers of this method intentionally use a very precise size for the serialized
      * data so a programmer making changes has to think when this size changes.
      * <p>
      *     In general, shrinking the over the wire size is great and the precise
@@ -108,10 +155,14 @@ public void testManyTypeConflictsWithParent() throws IOException {
      *     ESQL impossible to use at all for big mappings with many conflicts.
      * </p>
      */
-    private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) throws IOException {
-        EsIndex index = EsIndexSerializationTests.indexWithManyConflicts(withParent);
-        List<Attribute> attributes = Analyzer.mappingAsAttributes(randomSource(), index.mapping());
-        EsRelation relation = new EsRelation(randomSource(), index, attributes, IndexMode.STANDARD);
+    private void testSerializePlanWithIndex(EsIndex index, ByteSizeValue expected) throws IOException {
+        testSerializePlanWithIndex(index, expected, true);
+    }
+
+    private void testSerializePlanWithIndex(EsIndex index, ByteSizeValue expected, boolean keepAllFields) throws IOException {
+        List<Attribute> allAttributes = Analyzer.mappingAsAttributes(randomSource(), index.mapping());
+        List<Attribute> keepAttributes = keepAllFields ? allAttributes : List.of(allAttributes.get(0));
+        EsRelation relation = new EsRelation(randomSource(), index, keepAttributes, IndexMode.STANDARD);
         Limit limit = new Limit(randomSource(), new Literal(randomSource(), 10, DataType.INTEGER), relation);
         Project project = new Project(randomSource(), limit, limit.output());
         FragmentExec fragmentExec = new FragmentExec(project);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java
index 760d8a327ad20..cf90cf96fe683 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java
@@ -46,7 +46,7 @@ private static Analyzer makeAnalyzer(String mappingFileName) {
 
         return new Analyzer(
             new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, new EnrichResolution()),
-            new Verifier(new Metrics())
+            new Verifier(new Metrics(new EsqlFunctionRegistry()))
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
index da11a790e6f2f..5fbd5dd28050f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
@@ -125,14 +125,13 @@ private ComputeResponse randomResponse(boolean includeExecutionInfo) {
 
     public void testEmpty() {
         PlainActionFuture<ComputeResponse> results = new PlainActionFuture<>();
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(randomBoolean());
         try (
             ComputeListener ignored = ComputeListener.create(
                 RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                 transportService,
                 newTask(),
                 executionInfo,
-                System.nanoTime(),
                 results
             )
         ) {
@@ -145,14 +144,13 @@ public void testEmpty() {
     public void testCollectComputeResults() {
         PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
         List<DriverProfile> allProfiles = new ArrayList<>();
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(randomBoolean());
         try (
             ComputeListener computeListener = ComputeListener.create(
                 RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                 transportService,
                 newTask(),
                 executionInfo,
-                System.nanoTime(),
                 future
             )
         ) {
@@ -194,8 +192,9 @@ public void testAcquireComputeCCSListener() {
         PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
         List<DriverProfile> allProfiles = new ArrayList<>();
         String remoteAlias = "rc1";
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
         executionInfo.swapCluster(remoteAlias, (k, v) -> new EsqlExecutionInfo.Cluster(remoteAlias, "logs*", false));
+        executionInfo.markEndPlanning();  // set planning took time, so it can be used to calculate per-cluster took time
         try (
             ComputeListener computeListener = ComputeListener.create(
                 // 'whereRunning' for this test is the local cluster, waiting for a response from the remote cluster
@@ -203,7 +202,6 @@ public void testAcquireComputeCCSListener() {
                 transportService,
                 newTask(),
                 executionInfo,
-                System.nanoTime(),
                 future
             )
         ) {
@@ -239,6 +237,60 @@ public void testAcquireComputeCCSListener() {
         Mockito.verifyNoInteractions(transportService.getTaskManager());
     }
 
+    /**
+     * Tests the acquireCompute functionality running on the querying ("local") cluster, that is waiting upon
+     * a ComputeResponse from a remote cluster where we simulate connecting to a remote cluster running a version
+     * of ESQL that does not record and return CCS metadata. Ensure that the local cluster {@link EsqlExecutionInfo}
+     * is properly updated with took time and shard info is left unset.
+     */
+    public void testAcquireComputeCCSListenerWithComputeResponseFromOlderCluster() {
+        PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
+        List<DriverProfile> allProfiles = new ArrayList<>();
+        String remoteAlias = "rc1";
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
+        executionInfo.swapCluster(remoteAlias, (k, v) -> new EsqlExecutionInfo.Cluster(remoteAlias, "logs*", false));
+        executionInfo.markEndPlanning();  // set planning took time, so it can be used to calculate per-cluster took time
+        try (
+            ComputeListener computeListener = ComputeListener.create(
+                // 'whereRunning' for this test is the local cluster, waiting for a response from the remote cluster
+                RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                transportService,
+                newTask(),
+                executionInfo,
+                future
+            )
+        ) {
+            int tasks = randomIntBetween(1, 5);
+            for (int t = 0; t < tasks; t++) {
+                ComputeResponse resp = randomResponse(false); // older clusters will not return CCS metadata in response
+                allProfiles.addAll(resp.getProfiles());
+                // Use remoteAlias here to indicate what remote cluster alias the listener is waiting to hear back from
+                ActionListener<ComputeResponse> subListener = computeListener.acquireCompute(remoteAlias);
+                threadPool.schedule(
+                    ActionRunnable.wrap(subListener, l -> l.onResponse(resp)),
+                    TimeValue.timeValueNanos(between(0, 100)),
+                    threadPool.generic()
+                );
+            }
+        }
+        ComputeResponse response = future.actionGet(10, TimeUnit.SECONDS);
+        assertThat(
+            response.getProfiles().stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)),
+            equalTo(allProfiles.stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)))
+        );
+
+        assertTrue(executionInfo.isCrossClusterSearch());
+        EsqlExecutionInfo.Cluster rc1Cluster = executionInfo.getCluster(remoteAlias);
+        assertThat(rc1Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+        assertNull(rc1Cluster.getTotalShards());
+        assertNull(rc1Cluster.getSuccessfulShards());
+        assertNull(rc1Cluster.getSkippedShards());
+        assertNull(rc1Cluster.getFailedShards());
+        assertThat(rc1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+
+        Mockito.verifyNoInteractions(transportService.getTaskManager());
+    }
+
     /**
      * Run an acquireCompute cycle on the RemoteCluster.
      * AcquireCompute will fill in the took time on the EsqlExecutionInfo (the shard info is filled in before this,
@@ -248,7 +300,7 @@ public void testAcquireComputeCCSListener() {
     public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() {
         PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
         List<DriverProfile> allProfiles = new ArrayList<>();
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
         String remoteAlias = "rc1";
         executionInfo.swapCluster(
             remoteAlias,
@@ -271,7 +323,6 @@ public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() {
                 transportService,
                 newTask(),
                 executionInfo,
-                System.nanoTime(),
                 future
             )
         ) {
@@ -318,7 +369,7 @@ public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() {
     public void testAcquireComputeRunningOnQueryingClusterFillsInTookTime() {
         PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
         List<DriverProfile> allProfiles = new ArrayList<>();
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
         String localCluster = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
         // we need a remote cluster in the ExecutionInfo in order to simulate a CCS, since ExecutionInfo is only
         // fully filled in for cross-cluster searches
@@ -331,7 +382,6 @@ public void testAcquireComputeRunningOnQueryingClusterFillsInTookTime() {
                 transportService,
                 newTask(),
                 executionInfo,
-                System.nanoTime(),
                 future
             )
         ) {
@@ -372,14 +422,13 @@ public void testCancelOnFailure() throws Exception {
         int failedTasks = between(1, 100);
         PlainActionFuture<ComputeResponse> rootListener = new PlainActionFuture<>();
         CancellableTask rootTask = newTask();
-        EsqlExecutionInfo execInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo execInfo = new EsqlExecutionInfo(randomBoolean());
         try (
             ComputeListener computeListener = ComputeListener.create(
                 RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                 transportService,
                 rootTask,
                 execInfo,
-                System.nanoTime(),
                 rootListener
             )
         ) {
@@ -436,14 +485,13 @@ public void onFailure(Exception e) {
             }
         };
         CountDownLatch latch = new CountDownLatch(1);
-        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(randomBoolean());
         try (
             ComputeListener computeListener = ComputeListener.create(
                 RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                 transportService,
                 newTask(),
                 executionInfo,
-                System.nanoTime(),
                 ActionListener.runAfter(rootListener, latch::countDown)
             )
         ) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java
index 789d6e5adbfc7..4b9166c621940 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java
@@ -80,6 +80,18 @@ public void testColumnarWithAcceptText() {
         assertEquals(e.getMessage(), "Invalid use of [columnar] argument: cannot be used in combination with [txt, csv, tsv] formats");
     }
 
+    public void testIncludeCCSMetadataWithAcceptText() {
+        var accept = randomFrom("text/plain", "text/csv", "text/tab-separated-values");
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> getResponseMediaType(reqWithAccept(accept), createTestInstance(false, true))
+        );
+        assertEquals(
+            "Invalid use of [include_ccs_metadata] argument: cannot be used in combination with [txt, csv, tsv] formats",
+            e.getMessage()
+        );
+    }
+
     public void testColumnarWithParamText() {
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
@@ -88,6 +100,26 @@ public void testColumnarWithParamText() {
         assertEquals(e.getMessage(), "Invalid use of [columnar] argument: cannot be used in combination with [txt, csv, tsv] formats");
     }
 
+    public void testIncludeCCSMetadataWithNonJSONMediaTypesInParams() {
+        {
+            RestRequest restRequest = reqWithParams(Map.of("format", randomFrom("txt", "csv", "tsv")));
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> getResponseMediaType(restRequest, createTestInstance(false, true))
+            );
+            assertEquals(
+                "Invalid use of [include_ccs_metadata] argument: cannot be used in combination with [txt, csv, tsv] formats",
+                e.getMessage()
+            );
+        }
+        {
+            // check that no exception is thrown for the XContent types
+            RestRequest restRequest = reqWithParams(Map.of("format", randomFrom("SMILE", "YAML", "CBOR", "JSON")));
+            MediaType responseMediaType = getResponseMediaType(restRequest, createTestInstance(true, true));
+            assertNotNull(responseMediaType);
+        }
+    }
+
     public void testNoFormat() {
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
@@ -113,4 +145,10 @@ protected EsqlQueryRequest createTestInstance(boolean columnar) {
         request.columnar(columnar);
         return request;
     }
+
+    protected EsqlQueryRequest createTestInstance(boolean columnar, boolean includeCCSMetadata) {
+        var request = createTestInstance(columnar);
+        request.includeCCSMetadata(includeCCSMetadata);
+        return request;
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java
index 2ba397a3cb3de..95444c9b2423f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java
@@ -232,7 +232,7 @@ private Object randomValue() {
 
         private List<IndexableField> docFor(int i, Iterable<Object> values) {
             List<IndexableField> fields = new ArrayList<>();
-            fields.add(new LongField("i", i));
+            fields.add(new LongField("i", i, Field.Store.NO));
             fields.add(new TextField("str", "the quick brown fox jumped over the lazy dog", Field.Store.NO));
             switch (fieldType) {
                 case "long", "integer", "short", "byte" -> {
@@ -270,7 +270,10 @@ public List<List<Object>> build(RandomIndexWriter iw) throws IOException {
             List<List<Object>> fieldValues = new ArrayList<>(100);
             for (int i = 0; i < 100; i++) {
                 iw.addDocument(
-                    List.of(new LongField("i", i), new TextField("str", "the quick brown fox jumped over the lazy dog", Field.Store.NO))
+                    List.of(
+                        new LongField("i", i, Field.Store.NO),
+                        new TextField("str", "the quick brown fox jumped over the lazy dog", Field.Store.NO)
+                    )
                 );
                 fieldValues.add(List.of());
             }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
index 326756ad0b5f4..32b31cf78650b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
@@ -21,6 +21,7 @@
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class EsqlSessionTests extends ESTestCase {
 
@@ -30,7 +31,7 @@ public void testUpdateExecutionInfoWithUnavailableClusters() {
             final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
             final String remote1Alias = "remote1";
             final String remote2Alias = "remote2";
-            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true));
@@ -59,7 +60,7 @@ public void testUpdateExecutionInfoWithUnavailableClusters() {
             final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
             final String remote1Alias = "remote1";
             final String remote2Alias = "remote2";
-            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
@@ -87,7 +88,7 @@ public void testUpdateExecutionInfoWithUnavailableClusters() {
             final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
             final String remote1Alias = "remote1";
             final String remote2Alias = "remote2";
-            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
@@ -117,7 +118,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
             final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
             final String remote1Alias = "remote1";
             final String remote2Alias = "remote2";
-            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
@@ -160,7 +161,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
             final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
             final String remote1Alias = "remote1";
             final String remote2Alias = "remote2";
-            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
@@ -206,7 +207,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
             final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
             final String remote1Alias = "remote1";
             final String remote2Alias = "remote2";
-            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
@@ -243,6 +244,50 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
         }
     }
 
+    public void testUpdateExecutionInfoAtEndOfPlanning() {
+        String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+        String remote1Alias = "remote1";
+        String remote2Alias = "remote2";
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
+        executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+        executionInfo.swapCluster(
+            remote1Alias,
+            (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true, EsqlExecutionInfo.Cluster.Status.SKIPPED)
+        );
+        executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+        assertNull(executionInfo.planningTookTime());
+        assertNull(executionInfo.overallTook());
+        try {
+            Thread.sleep(1);
+        } catch (InterruptedException e) {}
+
+        EsqlSession.updateExecutionInfoAtEndOfPlanning(executionInfo);
+
+        assertThat(executionInfo.planningTookTime().millis(), greaterThanOrEqualTo(0L));
+        assertNull(executionInfo.overallTook());
+
+        // only remote1 should be altered, since it is the only one marked as SKIPPED when passed into updateExecutionInfoAtEndOfPlanning
+        EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+        assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING));
+        assertNull(localCluster.getTotalShards());
+        assertNull(localCluster.getTook());
+
+        EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
+        assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+        assertThat(remote1Cluster.getTotalShards(), equalTo(0));
+        assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0));
+        assertThat(remote1Cluster.getSkippedShards(), equalTo(0));
+        assertThat(remote1Cluster.getFailedShards(), equalTo(0));
+        assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+        assertThat(remote1Cluster.getTook().millis(), equalTo(executionInfo.planningTookTime().millis()));
+
+        EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
+        assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING));
+        assertNull(remote2Cluster.getTotalShards());
+        assertNull(remote2Cluster.getTook());
+    }
+
     private void assertClusterStatusAndHasNullCounts(EsqlExecutionInfo.Cluster cluster, EsqlExecutionInfo.Cluster.Status status) {
         assertThat(cluster.getStatus(), equalTo(status));
         assertNull(cluster.getTook());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
index adc449bfc092e..9edc85223e7b3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
@@ -120,7 +120,7 @@ public void testFailedMetric() {
             randomAlphaOfLength(10),
             EsqlTestUtils.TEST_CFG,
             enrichResolver,
-            new EsqlExecutionInfo(),
+            new EsqlExecutionInfo(randomBoolean()),
             groupIndicesByCluster,
             runPhase,
             new ActionListener<>() {
@@ -149,7 +149,7 @@ public void onFailure(Exception e) {
             randomAlphaOfLength(10),
             EsqlTestUtils.TEST_CFG,
             enrichResolver,
-            new EsqlExecutionInfo(),
+            new EsqlExecutionInfo(randomBoolean()),
             groupIndicesByCluster,
             runPhase,
             new ActionListener<>() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java
index 203e5c3bd37ee..5e6588d2295f9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java
@@ -10,9 +10,14 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer;
@@ -32,6 +37,7 @@
 import static org.elasticsearch.xpack.esql.stats.FeatureMetric.STATS;
 import static org.elasticsearch.xpack.esql.stats.FeatureMetric.WHERE;
 import static org.elasticsearch.xpack.esql.stats.Metrics.FPREFIX;
+import static org.elasticsearch.xpack.esql.stats.Metrics.FUNC_PREFIX;
 
 public class VerifierMetricsTests extends ESTestCase {
 
@@ -54,6 +60,8 @@ public void testDissectQuery() {
         assertEquals(0, drop(c));
         assertEquals(0, keep(c));
         assertEquals(0, rename(c));
+
+        assertEquals(1, function("concat", c));
     }
 
     public void testEvalQuery() {
@@ -73,6 +81,8 @@ public void testEvalQuery() {
         assertEquals(0, drop(c));
         assertEquals(0, keep(c));
         assertEquals(0, rename(c));
+
+        assertEquals(1, function("length", c));
     }
 
     public void testGrokQuery() {
@@ -92,6 +102,8 @@ public void testGrokQuery() {
         assertEquals(0, drop(c));
         assertEquals(0, keep(c));
         assertEquals(0, rename(c));
+
+        assertEquals(1, function("concat", c));
     }
 
     public void testLimitQuery() {
@@ -149,6 +161,8 @@ public void testStatsQuery() {
         assertEquals(0, drop(c));
         assertEquals(0, keep(c));
         assertEquals(0, rename(c));
+
+        assertEquals(1, function("max", c));
     }
 
     public void testWhereQuery() {
@@ -190,7 +204,7 @@ public void testTwoWhereQuery() {
     }
 
     public void testTwoQueriesExecuted() {
-        Metrics metrics = new Metrics();
+        Metrics metrics = new Metrics(new EsqlFunctionRegistry());
         Verifier verifier = new Verifier(metrics);
         esqlWithVerifier("""
                from employees
@@ -226,6 +240,64 @@ public void testTwoQueriesExecuted() {
         assertEquals(0, drop(c));
         assertEquals(0, keep(c));
         assertEquals(0, rename(c));
+
+        assertEquals(1, function("length", c));
+        assertEquals(1, function("concat", c));
+        assertEquals(1, function("max", c));
+        assertEquals(1, function("min", c));
+
+        assertEquals(0, function("sin", c));
+        assertEquals(0, function("cos", c));
+    }
+
+    public void testMultipleFunctions() {
+        Metrics metrics = new Metrics(new EsqlFunctionRegistry());
+        Verifier verifier = new Verifier(metrics);
+        esqlWithVerifier("""
+               from employees
+               | where languages > 2
+               | limit 5
+               | eval name_len = length(first_name), surname_len = length(last_name)
+               | sort length(first_name)
+               | limit 3
+            """, verifier);
+
+        Counters c = metrics.stats();
+        assertEquals(1, function("length", c));
+        assertEquals(0, function("concat", c));
+
+        esqlWithVerifier("""
+              from employees
+              | where languages > 2
+              | sort first_name desc nulls first
+              | dissect concat(first_name, " ", last_name) "%{a} %{b}"
+              | grok concat(first_name, " ", last_name) "%{WORD:a} %{WORD:b}"
+              | eval name_len = length(first_name), surname_len = length(last_name)
+              | stats x = max(languages)
+              | sort x
+              | stats y = min(x) by x
+            """, verifier);
+        c = metrics.stats();
+
+        assertEquals(2, function("length", c));
+        assertEquals(1, function("concat", c));
+        assertEquals(1, function("max", c));
+        assertEquals(1, function("min", c));
+
+        EsqlFunctionRegistry fr = new EsqlFunctionRegistry().snapshotRegistry();
+        Map<Class<?>, String> functions = new HashMap<>();
+        for (FunctionDefinition func : fr.listFunctions()) {
+            if (functions.containsKey(func.clazz()) == false) {
+                functions.put(func.clazz(), func.name());
+            }
+        }
+        for (String value : functions.values()) {
+            if (Set.of("length", "concat", "max", "min").contains(value) == false) {
+                assertEquals(0, function(value, c));
+            }
+        }
+        Map<?, ?> map = (Map<?, ?>) c.toNestedMap().get("functions");
+        assertEquals(functions.size(), map.size());
     }
 
     public void testEnrich() {
@@ -251,6 +323,8 @@ public void testEnrich() {
         assertEquals(0, drop(c));
         assertEquals(1L, keep(c));
         assertEquals(0, rename(c));
+
+        assertEquals(1, function("to_string", c));
     }
 
     public void testMvExpand() {
@@ -298,6 +372,8 @@ public void testShowInfo() {
         assertEquals(0, drop(c));
         assertEquals(0, keep(c));
         assertEquals(0, rename(c));
+
+        assertEquals(1, function("count", c));
     }
 
     public void testRow() {
@@ -336,6 +412,8 @@ public void testDropAndRename() {
         assertEquals(1L, drop(c));
         assertEquals(0, keep(c));
         assertEquals(1L, rename(c));
+
+        assertEquals(1, function("count", c));
     }
 
     public void testKeep() {
@@ -422,6 +500,19 @@ private long rename(Counters c) {
         return c.get(FPREFIX + RENAME);
     }
 
+    private long function(String function, Counters c) {
+        return c.get(FUNC_PREFIX + function);
+    }
+
+    private void assertNullFunction(String function, Counters c) {
+        try {
+            c.get(FUNC_PREFIX + function);
+            fail();
+        } catch (NullPointerException npe) {
+
+        }
+    }
+
     private Counters esql(String esql) {
         return esql(esql, null);
     }
@@ -434,7 +525,7 @@ private Counters esql(String esql, Verifier v) {
         Verifier verifier = v;
         Metrics metrics = null;
         if (v == null) {
-            metrics = new Metrics();
+            metrics = new Metrics(new EsqlFunctionRegistry());
             verifier = new Verifier(metrics);
         }
         analyzer(verifier).analyze(parser.createStatement(esql));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
index d186b4c199d77..2bee0188b9fab 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
@@ -21,13 +21,13 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttributeTests;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedNamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.esql.core.tree.AbstractNodeTestCase;
 import org.elasticsearch.xpack.esql.core.tree.Node;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -164,6 +164,16 @@ public void testInfoParameters() throws Exception {
          * in the parameters and not included.
          */
         expectedCount -= 1;
+
+        // special exceptions with private constructors
+        if (MetadataAttribute.class.equals(subclass) || ReferenceAttribute.class.equals(subclass)) {
+            expectedCount++;
+        }
+
+        if (FieldAttribute.class.equals(subclass)) {
+            expectedCount += 2;
+        }
+
         assertEquals(expectedCount, info(node).properties().size());
     }
 
@@ -174,6 +184,9 @@ public void testInfoParameters() throws Exception {
      * implementations in the process.
      */
     public void testTransform() throws Exception {
+        if (FieldAttribute.class.equals(subclass)) {
+            assumeTrue("FieldAttribute private constructor", false);
+        }
         Constructor<T> ctor = longestCtor(subclass);
         Object[] nodeCtorArgs = ctorArgs(ctor);
         T node = ctor.newInstance(nodeCtorArgs);
@@ -407,12 +420,6 @@ public void accept(Page page) {
                 }
                 return b.toString();
             }
-        } else if (toBuildClass == Like.class) {
-
-            if (argClass == LikePattern.class) {
-                return new LikePattern(randomAlphaOfLength(16), randomFrom('\\', '|', '/', '`'));
-            }
-
         } else if (argClass == Dissect.Parser.class) {
             // Dissect.Parser is a record / final, cannot be mocked
             String pattern = randomDissectPattern();
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
index 8ad083683f696..babb9fc8c0bd1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
@@ -62,8 +62,7 @@ public void testCommonTypeNull() {
     }
 
     public void testCommonTypeStrings() {
-        List<DataType> STRINGS = Arrays.stream(DataType.values()).filter(DataType::isString).toList();
-        for (DataType dataType1 : STRINGS) {
+        for (DataType dataType1 : DataType.stringTypes()) {
             for (DataType dataType2 : DataType.values()) {
                 if (dataType2 == NULL) {
                     assertEqualsCommonType(dataType1, NULL, dataType1);
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java
index a6c369734f0e3..a79424b8b7d59 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.fleet.rest;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.client.internal.node.NodeClient;
@@ -57,12 +56,11 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        SearchRequest searchRequest;
-        if (request.hasParam("min_compatible_shard_node")) {
-            searchRequest = new SearchRequest(Version.fromString(request.param("min_compatible_shard_node")));
-        } else {
-            searchRequest = new SearchRequest();
-        }
+        SearchRequest searchRequest = new SearchRequest();
+        // access the BwC param, but just drop it
+        // this might be set by old clients
+        request.param("min_compatible_shard_node");
+
         String[] indices = searchRequest.indices();
         if (indices.length > 1) {
             throw new IllegalArgumentException(
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
index 081061fd9ceac..f07d17fee87f5 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
@@ -6,27 +6,20 @@
  */
 package org.elasticsearch.xpack.frozen.rest.action;
 
-import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
-import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.protocol.xpack.frozen.FreezeRequest;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
 
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.RestStatus.GONE;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
@@ -46,7 +39,6 @@ public final class RestFreezeIndexAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(POST, "/{index}/_freeze").deprecated(FREEZE_REMOVED, RestApiVersion.V_7).build(),
             // Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build()
             Route.builder(POST, "/{index}/_unfreeze").deprecateAndKeep(UNFREEZE_DEPRECATED).build()
         );
@@ -54,22 +46,6 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.path().endsWith("/_freeze")) {
-            // translate to a get indices request, so that we'll 404 on non-existent indices
-            final GetIndexRequest getIndexRequest = new GetIndexRequest();
-            getIndexRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
-            getIndexRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-            getIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, getIndexRequest.indicesOptions()));
-            return channel -> client.admin().indices().getIndex(getIndexRequest, new RestBuilderListener<>(channel) {
-                @Override
-                public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBuilder builder) throws Exception {
-                    builder.close();
-                    // but if the index *does* exist, we still just respond with 410 -- there's no such thing as _freeze anymore
-                    return new RestResponse(channel, GONE, new UnsupportedOperationException(FREEZE_REMOVED));
-                }
-            });
-        }
-
         final var freezeRequest = new FreezeRequest(
             getMasterNodeTimeout(request),
             getAckTimeout(request),
diff --git a/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java b/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java
index 3623d3671e83f..6d90b0e67ee83 100644
--- a/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java
+++ b/x-pack/plugin/graph/src/internalClusterTest/java/org/elasticsearch/xpack/graph/test/GraphTests.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.graph.test;
 
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
 import org.elasticsearch.action.admin.indices.segments.ShardSegments;
@@ -165,7 +165,7 @@ public void testLargeNumberTermsStartCrawl() {
         VertexRequest peopleNames = hop1.addVertexRequest("people").minDocCount(1);
         peopleNames.addInclude("john", 1);
 
-        for (int i = 0; i < BooleanQuery.getMaxClauseCount() + 1; i++) {
+        for (int i = 0; i < IndexSearcher.getMaxClauseCount() + 1; i++) {
             peopleNames.addInclude("unknown" + i, 1);
         }
 
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
index 36e8eaf94c8be..b60ce13e0228c 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PriorityQueue;
 import org.elasticsearch.ExceptionsHelper;
@@ -564,7 +564,7 @@ private static void addBigOrClause(Map<String, Set<Vertex>> lastHopFindings, Boo
             for (Entry<String, Set<Vertex>> entry : lastHopFindings.entrySet()) {
                 numClauses += entry.getValue().size();
             }
-            if (numClauses < BooleanQuery.getMaxClauseCount()) {
+            if (numClauses < IndexSearcher.getMaxClauseCount()) {
                 // We can afford to build a Boolean OR query with individual
                 // boosts for interesting terms
                 for (Entry<String, Set<Vertex>> entry : lastHopFindings.entrySet()) {
@@ -755,7 +755,7 @@ private double getInitialTotalSignalStrength(Hop rootHop, Sampler sample) {
         private static void addNormalizedBoosts(BoolQueryBuilder includesContainer, VertexRequest vr) {
             TermBoost[] termBoosts = vr.includeValues();
 
-            if ((includesContainer.should().size() + termBoosts.length) > BooleanQuery.getMaxClauseCount()) {
+            if ((includesContainer.should().size() + termBoosts.length) > IndexSearcher.getMaxClauseCount()) {
                 // Too many terms - we need a cheaper form of query to execute this
                 List<String> termValues = new ArrayList<>();
                 for (TermBoost tb : termBoosts) {
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
index 472b6f8087a56..973151e73d8dc 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest;
 import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest.TermBoost;
@@ -68,22 +67,7 @@ public class RestGraphAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/{index}/_graph/explore")
-                .replaces(GET, "/{index}" + URI_BASE + "/graph/_explore", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/{index}/_graph/explore")
-                .replaces(POST, "/{index}" + URI_BASE + "/graph/_explore", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, "/{index}/{type}/_graph/explore").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(GET, "/{index}/{type}" + URI_BASE + "/graph/_explore")
-                .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/{index}/{type}/_graph/explore").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(),
-            Route.builder(POST, "/{index}/{type}" + URI_BASE + "/graph/_explore")
-                .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, "/{index}/_graph/explore"), new Route(POST, "/{index}/_graph/explore"));
     }
 
     @Override
@@ -93,11 +77,6 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) {
-            deprecationLogger.compatibleCritical("graph_with_types", TYPES_DEPRECATION_MESSAGE);
-            request.param("type");
-        }
-
         GraphExploreRequest graphRequest = new GraphExploreRequest(Strings.splitStringByCommaToArray(request.param("index")));
         graphRequest.indicesOptions(IndicesOptions.fromRequest(request, graphRequest.indicesOptions()));
         graphRequest.routing(request.param("routing"));
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java
index ec8f7c230b1d3..9b7262e8f9b32 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java
@@ -34,6 +34,7 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.TimeSeriesRestDriver.createFullPolicy;
@@ -307,14 +308,16 @@ public void testStepInfoPreservedOnAutoRetry() throws Exception {
 
         assertBusy(() -> {
             Map<String, Object> explainIndex = explainIndex(client(), indexName);
-            assertThat(explainIndex.get("failed_step_retry_count"), notNullValue());
-            assertThat(explainIndex.get("previous_step_info"), notNullValue());
-            assertThat((int) explainIndex.get("failed_step_retry_count"), greaterThan(0));
+            var assertionMessage = "Assertion failed for the following response: " + explainIndex;
+            assertThat(assertionMessage, explainIndex.get("failed_step_retry_count"), notNullValue());
+            assertThat(assertionMessage, explainIndex.get("previous_step_info"), notNullValue());
+            assertThat(assertionMessage, (int) explainIndex.get("failed_step_retry_count"), greaterThan(0));
             assertThat(
+                assertionMessage,
                 explainIndex.get("previous_step_info").toString(),
                 containsString("rollover_alias [" + aliasName + "] does not point to index [" + indexName + "]")
             );
-        });
+        }, 30, TimeUnit.SECONDS);
     }
 
     private void assertUnmanagedIndex(Map<String, Object> explainIndexMap) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java
index b3f29535020bf..2499cd92113c2 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java
@@ -137,7 +137,8 @@ static ClusterState moveClusterStateToStep(
             lifecycleState,
             newStepKey,
             nowSupplier,
-            forcePhaseDefinitionRefresh
+            forcePhaseDefinitionRefresh,
+            true
         );
 
         return LifecycleExecutionStateUtils.newClusterStateWithLifecycleState(state, idxMeta.getIndex(), newLifecycleState);
@@ -175,6 +176,7 @@ static ClusterState moveClusterStateToErrorStep(
             currentState,
             new Step.StepKey(currentStep.phase(), currentStep.action(), ErrorStep.NAME),
             nowSupplier,
+            false,
             false
         );
 
@@ -243,7 +245,8 @@ static ClusterState moveClusterStateToPreviouslyFailedStep(
                 lifecycleState,
                 nextStepKey,
                 nowSupplier,
-                forcePhaseDefinitionRefresh
+                forcePhaseDefinitionRefresh,
+                false
             );
 
             LifecycleExecutionState.Builder retryStepState = LifecycleExecutionState.builder(nextStepState);
@@ -277,7 +280,8 @@ private static LifecycleExecutionState updateExecutionStateToStep(
         LifecycleExecutionState existingState,
         Step.StepKey newStep,
         LongSupplier nowSupplier,
-        boolean forcePhaseDefinitionRefresh
+        boolean forcePhaseDefinitionRefresh,
+        boolean allowNullPreviousStepInfo
     ) {
         Step.StepKey currentStep = Step.getCurrentStepKey(existingState);
         long nowAsMillis = nowSupplier.getAsLong();
@@ -289,7 +293,9 @@ private static LifecycleExecutionState updateExecutionStateToStep(
 
         // clear any step info or error-related settings from the current step
         updatedState.setFailedStep(null);
-        updatedState.setPreviousStepInfo(existingState.stepInfo());
+        if (allowNullPreviousStepInfo || existingState.stepInfo() != null) {
+            updatedState.setPreviousStepInfo(existingState.stepInfo());
+        }
         updatedState.setStepInfo(null);
         updatedState.setIsAutoRetryableError(null);
         updatedState.setFailedStepRetryCount(null);
@@ -390,7 +396,7 @@ public static LifecycleExecutionState moveStateToNextActionAndUpdateCachedPhase(
         updatedState.setStep(nextStep.name());
         updatedState.setStepTime(nowAsMillis);
         updatedState.setFailedStep(null);
-        updatedState.setPreviousStepInfo(existingState.stepInfo());
+        updatedState.setPreviousStepInfo(null);
         updatedState.setStepInfo(null);
         updatedState.setIsAutoRetryableError(null);
         updatedState.setFailedStepRetryCount(null);
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java
index 209839c9d24df..eceb81542377a 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java
@@ -390,7 +390,10 @@ public void doTestExceptionStillProcessesOtherIndices(boolean useOnMaster) {
         MockAction mockAction = new MockAction(Collections.singletonList(i2mockStep));
         Phase i2phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
         LifecyclePolicy i2policy = newTestLifecyclePolicy(policy1, Collections.singletonMap(i2phase.getName(), i1phase));
-        Index index2 = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Index index2 = new Index(
+            randomValueOtherThan(index1.getName(), () -> randomAlphaOfLengthBetween(1, 20)),
+            randomAlphaOfLengthBetween(1, 20)
+        );
         LifecycleExecutionState.Builder i2lifecycleState = LifecycleExecutionState.builder();
         i2lifecycleState.setPhase(i2currentStepKey.phase());
         i2lifecycleState.setAction(i2currentStepKey.action());
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java
index 3f3285c5c2bd7..bcd6026618a05 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.ilm.action;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.repositories.reservedstate.ReservedRepositoryAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterModule;
@@ -22,6 +21,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
@@ -30,6 +30,7 @@
 import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask;
 import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor;
 import org.elasticsearch.reservedstate.service.ReservedStateVersion;
+import org.elasticsearch.reservedstate.service.ReservedStateVersionCheck;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
@@ -362,7 +363,7 @@ public void testOperatorControllerFromJSONContent() throws IOException {
         AtomicReference<Exception> x = new AtomicReference<>();
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) {
-            controller.process("operator", parser, x::set);
+            controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), x::set);
 
             assertThat(x.get(), instanceOf(IllegalStateException.class));
             assertThat(x.get().getMessage(), containsString("Error processing state change request for operator"));
@@ -383,7 +384,7 @@ public void testOperatorControllerFromJSONContent() throws IOException {
         );
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) {
-            controller.process("operator", parser, Assert::assertNull);
+            controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), Assert::assertNull);
         }
     }
 
@@ -417,10 +418,10 @@ public void testOperatorControllerWithPluginPackage() {
                     )
                 )
             ),
-            new ReservedStateVersion(123L, Version.CURRENT)
+            new ReservedStateVersion(123L, BuildVersion.current())
         );
 
-        controller.process("operator", pack, x::set);
+        controller.process("operator", pack, randomFrom(ReservedStateVersionCheck.values()), x::set);
 
         assertThat(x.get(), instanceOf(IllegalStateException.class));
         assertThat(x.get().getMessage(), containsString("Error processing state change request for operator"));
@@ -439,6 +440,6 @@ public void testOperatorControllerWithPluginPackage() {
             )
         );
 
-        controller.process("operator", pack, Assert::assertNull);
+        controller.process("operator", pack, randomFrom(ReservedStateVersionCheck.values()), Assert::assertNull);
     }
 }
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java
new file mode 100644
index 0000000000000..273b16d295a3d
--- /dev/null
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java
@@ -0,0 +1,232 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.inference.TaskType;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+public class CreateFromDeploymentIT extends InferenceBaseRestTest {
+
+    @SuppressWarnings("unchecked")
+    public void testAttachToDeployment() throws IOException {
+        var modelId = "attach_to_deployment";
+        var deploymentId = "existing_deployment";
+
+        CustomElandModelIT.createMlNodeTextExpansionModel(modelId, client());
+        var response = startMlNodeDeploymemnt(modelId, deploymentId);
+        assertOkOrCreated(response);
+
+        var inferenceId = "inference_on_existing_deployment";
+        var putModel = putModel(inferenceId, endpointConfig(deploymentId), TaskType.SPARSE_EMBEDDING);
+        var serviceSettings = putModel.get("service_settings");
+        assertThat(
+            putModel.toString(),
+            serviceSettings,
+            is(Map.of("num_allocations", 1, "num_threads", 1, "model_id", "attach_to_deployment", "deployment_id", "existing_deployment"))
+        );
+
+        var results = infer(inferenceId, List.of("washing machine"));
+        assertNotNull(results.get("sparse_embedding"));
+
+        deleteModel(inferenceId);
+        // assert deployment not stopped
+        var stats = (List<Map<String, Object>>) getTrainedModelStats(modelId).get("trained_model_stats");
+        var deploymentStats = stats.get(0).get("deployment_stats");
+        assertNotNull(stats.toString(), deploymentStats);
+
+        stopMlNodeDeployment(deploymentId);
+    }
+
+    public void testAttachWithModelId() throws IOException {
+        var modelId = "attach_with_model_id";
+        var deploymentId = "existing_deployment_with_model_id";
+
+        CustomElandModelIT.createMlNodeTextExpansionModel(modelId, client());
+        var response = startMlNodeDeploymemnt(modelId, deploymentId);
+        assertOkOrCreated(response);
+
+        var inferenceId = "inference_on_existing_deployment";
+        var putModel = putModel(inferenceId, endpointConfig(modelId, deploymentId), TaskType.SPARSE_EMBEDDING);
+        var serviceSettings = putModel.get("service_settings");
+        assertThat(
+            putModel.toString(),
+            serviceSettings,
+            is(
+                Map.of(
+                    "num_allocations",
+                    1,
+                    "num_threads",
+                    1,
+                    "model_id",
+                    "attach_with_model_id",
+                    "deployment_id",
+                    "existing_deployment_with_model_id"
+                )
+            )
+        );
+
+        var results = infer(inferenceId, List.of("washing machine"));
+        assertNotNull(results.get("sparse_embedding"));
+
+        stopMlNodeDeployment(deploymentId);
+    }
+
+    public void testModelIdDoesNotMatch() throws IOException {
+        var modelId = "attach_with_model_id";
+        var deploymentId = "existing_deployment_with_model_id";
+        var aDifferentModelId = "not_the_same_as_the_one_used_in_the_deployment";
+
+        CustomElandModelIT.createMlNodeTextExpansionModel(modelId, client());
+        var response = startMlNodeDeploymemnt(modelId, deploymentId);
+        assertOkOrCreated(response);
+
+        var inferenceId = "inference_on_existing_deployment";
+        var e = expectThrows(
+            ResponseException.class,
+            () -> putModel(inferenceId, endpointConfig(aDifferentModelId, deploymentId), TaskType.SPARSE_EMBEDDING)
+        );
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "Deployment [existing_deployment_with_model_id] uses model [attach_with_model_id] "
+                    + "which does not match the model [not_the_same_as_the_one_used_in_the_deployment] in the request."
+            )
+        );
+    }
+
+    public void testDeploymentDoesNotExist() {
+        var deploymentId = "missing_deployment";
+
+        var inferenceId = "inference_on_missing_deployment";
+        var e = expectThrows(ResponseException.class, () -> putModel(inferenceId, endpointConfig(deploymentId), TaskType.SPARSE_EMBEDDING));
+        assertThat(e.getMessage(), containsString("Cannot find deployment [missing_deployment]"));
+    }
+
+    public void testNumAllocationsIsUpdated() throws IOException {
+        var modelId = "update_num_allocations";
+        var deploymentId = modelId;
+
+        CustomElandModelIT.createMlNodeTextExpansionModel(modelId, client());
+        var response = startMlNodeDeploymemnt(modelId, deploymentId);
+        assertOkOrCreated(response);
+
+        var inferenceId = "test_num_allocations_updated";
+        var putModel = putModel(inferenceId, endpointConfig(deploymentId), TaskType.SPARSE_EMBEDDING);
+        var serviceSettings = putModel.get("service_settings");
+        assertThat(
+            putModel.toString(),
+            serviceSettings,
+            is(
+                Map.of(
+                    "num_allocations",
+                    1,
+                    "num_threads",
+                    1,
+                    "model_id",
+                    "update_num_allocations",
+                    "deployment_id",
+                    "update_num_allocations"
+                )
+            )
+        );
+
+        assertOkOrCreated(updateMlNodeDeploymemnt(deploymentId, 2));
+
+        var updatedServiceSettings = getModel(inferenceId).get("service_settings");
+        assertThat(
+            updatedServiceSettings.toString(),
+            updatedServiceSettings,
+            is(
+                Map.of(
+                    "num_allocations",
+                    2,
+                    "num_threads",
+                    1,
+                    "model_id",
+                    "update_num_allocations",
+                    "deployment_id",
+                    "update_num_allocations"
+                )
+            )
+        );
+    }
+
+    private String endpointConfig(String deploymentId) {
+        return Strings.format("""
+            {
+              "service": "elasticsearch",
+              "service_settings": {
+                "deployment_id": "%s"
+              }
+            }
+            """, deploymentId);
+    }
+
+    private String endpointConfig(String modelId, String deploymentId) {
+        return Strings.format("""
+            {
+              "service": "elasticsearch",
+              "service_settings": {
+                "model_id": "%s",
+                "deployment_id": "%s"
+              }
+            }
+            """, modelId, deploymentId);
+    }
+
+    private Response startMlNodeDeploymemnt(String modelId, String deploymentId) throws IOException {
+        String endPoint = "/_ml/trained_models/"
+            + modelId
+            + "/deployment/_start?timeout=10s&wait_for=started"
+            + "&threads_per_allocation=1"
+            + "&number_of_allocations=1";
+
+        if (deploymentId != null) {
+            endPoint = endPoint + "&deployment_id=" + deploymentId;
+        }
+
+        Request request = new Request("POST", endPoint);
+        return client().performRequest(request);
+    }
+
+    private Response updateMlNodeDeploymemnt(String deploymentId, int numAllocations) throws IOException {
+        String endPoint = "/_ml/trained_models/" + deploymentId + "/deployment/_update";
+
+        var body = Strings.format("""
+            {
+              "number_of_allocations": %d
+            }
+            """, numAllocations);
+
+        Request request = new Request("POST", endPoint);
+        request.setJsonEntity(body);
+        return client().performRequest(request);
+    }
+
+    protected void stopMlNodeDeployment(String deploymentId) throws IOException {
+        String endpoint = "/_ml/trained_models/" + deploymentId + "/deployment/_stop";
+        Request request = new Request("POST", endpoint);
+        request.addParameter("force", "true");
+        client().performRequest(request);
+    }
+
+    protected Map<String, Object> getTrainedModelStats(String modelId) throws IOException {
+        Request request = new Request("GET", "/_ml/trained_models/" + modelId + "/_stats");
+        return entityAsMap(client().performRequest(request));
+    }
+}
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java
index c05d08fa33692..e6d959bafea3f 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.inference.TaskType;
 
@@ -65,11 +66,12 @@ public class CustomElandModelIT extends InferenceBaseRestTest {
     public void testSparse() throws IOException {
         String modelId = "custom-text-expansion-model";
 
-        createTextExpansionModel(modelId);
-        putModelDefinition(modelId, BASE_64_ENCODED_MODEL, RAW_MODEL_SIZE);
+        createTextExpansionModel(modelId, client());
+        putModelDefinition(modelId, BASE_64_ENCODED_MODEL, RAW_MODEL_SIZE, client());
         putVocabulary(
             List.of("these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"),
-            modelId
+            modelId,
+            client()
         );
 
         var inferenceConfig = """
@@ -90,7 +92,7 @@ public void testSparse() throws IOException {
         assertNotNull(results.get("sparse_embedding"));
     }
 
-    protected void createTextExpansionModel(String modelId) throws IOException {
+    static void createTextExpansionModel(String modelId, RestClient client) throws IOException {
         // with_special_tokens: false for this test with limited vocab
         Request request = new Request("PUT", "/_ml/trained_models/" + modelId);
         request.setJsonEntity("""
@@ -107,10 +109,10 @@ protected void createTextExpansionModel(String modelId) throws IOException {
                  }
                }
              }""");
-        client().performRequest(request);
+        client.performRequest(request);
     }
 
-    protected void putVocabulary(List<String> vocabulary, String modelId) throws IOException {
+    static void putVocabulary(List<String> vocabulary, String modelId, RestClient client) throws IOException {
         List<String> vocabularyWithPad = new ArrayList<>();
         vocabularyWithPad.add("[PAD]");
         vocabularyWithPad.add("[UNK]");
@@ -121,14 +123,27 @@ protected void putVocabulary(List<String> vocabulary, String modelId) throws IOE
         request.setJsonEntity(Strings.format("""
             { "vocabulary": [%s] }
             """, quotedWords));
-        client().performRequest(request);
+        client.performRequest(request);
     }
 
-    protected void putModelDefinition(String modelId, String base64EncodedModel, long unencodedModelSize) throws IOException {
+    static void putModelDefinition(String modelId, String base64EncodedModel, long unencodedModelSize, RestClient client)
+        throws IOException {
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/definition/0");
         String body = Strings.format("""
             {"total_definition_length":%s,"definition": "%s","total_parts": 1}""", unencodedModelSize, base64EncodedModel);
         request.setJsonEntity(body);
-        client().performRequest(request);
+        client.performRequest(request);
     }
+
+    // Create the model including definition and vocab
+    static void createMlNodeTextExpansionModel(String modelId, RestClient client) throws IOException {
+        createTextExpansionModel(modelId, client);
+        putModelDefinition(modelId, BASE_64_ENCODED_MODEL, RAW_MODEL_SIZE, client);
+        putVocabulary(
+            List.of("these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"),
+            modelId,
+            client
+        );
+    }
+
 }
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java
similarity index 55%
rename from x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java
rename to x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java
index 5d84aad4b7344..3a774a7a37d93 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java
@@ -22,13 +22,13 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.oneOf;
 
-public class DefaultElserIT extends InferenceBaseRestTest {
+public class DefaultEndPointsIT extends InferenceBaseRestTest {
 
     private TestThreadPool threadPool;
 
     @Before
     public void createThreadPool() {
-        threadPool = new TestThreadPool(DefaultElserIT.class.getSimpleName());
+        threadPool = new TestThreadPool(DefaultEndPointsIT.class.getSimpleName());
     }
 
     @After
@@ -38,7 +38,7 @@ public void tearDown() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
-    public void testInferCreatesDefaultElser() throws IOException {
+    public void testInferDeploysDefaultElser() throws IOException {
         assumeTrue("Default config requires a feature flag", DefaultElserFeatureFlag.isEnabled());
         var model = getModel(ElasticsearchInternalService.DEFAULT_ELSER_ID);
         assertDefaultElserConfig(model);
@@ -64,7 +64,42 @@ private static void assertDefaultElserConfig(Map<String, Object> modelConfig) {
         assertThat(
             modelConfig.toString(),
             adaptiveAllocations,
-            Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 1, "max_number_of_allocations", 8))
+            Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8))
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testInferDeploysDefaultE5() throws IOException {
+        assumeTrue("Default config requires a feature flag", DefaultElserFeatureFlag.isEnabled());
+        var model = getModel(ElasticsearchInternalService.DEFAULT_E5_ID);
+        assertDefaultE5Config(model);
+
+        var inputs = List.of("Hello World", "Goodnight moon");
+        var queryParams = Map.of("timeout", "120s");
+        var results = infer(ElasticsearchInternalService.DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, inputs, queryParams);
+        var embeddings = (List<Map<String, Object>>) results.get("text_embedding");
+        assertThat(results.toString(), embeddings, hasSize(2));
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertDefaultE5Config(Map<String, Object> modelConfig) {
+        assertEquals(modelConfig.toString(), ElasticsearchInternalService.DEFAULT_E5_ID, modelConfig.get("inference_id"));
+        assertEquals(modelConfig.toString(), ElasticsearchInternalService.NAME, modelConfig.get("service"));
+        assertEquals(modelConfig.toString(), TaskType.TEXT_EMBEDDING.toString(), modelConfig.get("task_type"));
+
+        var serviceSettings = (Map<String, Object>) modelConfig.get("service_settings");
+        assertThat(
+            modelConfig.toString(),
+            serviceSettings.get("model_id"),
+            is(oneOf(".multilingual-e5-small", ".multilingual-e5-small_linux-x86_64"))
+        );
+        assertEquals(modelConfig.toString(), 1, serviceSettings.get("num_threads"));
+
+        var adaptiveAllocations = (Map<String, Object>) serviceSettings.get("adaptive_allocations");
+        assertThat(
+            modelConfig.toString(),
+            adaptiveAllocations,
+            Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 8))
         );
     }
 }
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
index f82b6f155c0a0..74c1e2f0d3356 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
@@ -81,6 +81,21 @@ static String mockSparseServiceModelConfig(@Nullable TaskType taskTypeInBody) {
             """, taskType);
     }
 
+    static String updateConfig(@Nullable TaskType taskTypeInBody, String apiKey, int temperature) {
+        var taskType = taskTypeInBody == null ? "" : "\"task_type\": \"" + taskTypeInBody + "\",";
+        return Strings.format("""
+            {
+              %s
+              "service_settings": {
+                "api_key": "%s"
+              },
+              "task_settings": {
+                "temperature": %d
+              }
+            }
+            """, taskType, apiKey, temperature);
+    }
+
     static String mockCompletionServiceModelConfig(@Nullable TaskType taskTypeInBody) {
         var taskType = taskTypeInBody == null ? "" : "\"task_type\": \"" + taskTypeInBody + "\",";
         return Strings.format("""
@@ -192,7 +207,12 @@ protected void putSemanticText(String endpointId, String searchEndpointId, Strin
     }
 
     protected Map<String, Object> putModel(String modelId, String modelConfig, TaskType taskType) throws IOException {
-        String endpoint = Strings.format("_inference/%s/%s", taskType, modelId);
+        String endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, modelId);
+        return putRequest(endpoint, modelConfig);
+    }
+
+    protected Map<String, Object> updateEndpoint(String inferenceID, String modelConfig, TaskType taskType) throws IOException {
+        String endpoint = Strings.format("_inference/%s/%s/_update", taskType, inferenceID);
         return putRequest(endpoint, modelConfig);
     }
 
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
index 5a84fd8985504..37de2caadb475 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
@@ -16,12 +16,15 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
 import static org.hamcrest.Matchers.hasSize;
@@ -29,7 +32,7 @@
 public class InferenceCrudIT extends InferenceBaseRestTest {
 
     @SuppressWarnings("unchecked")
-    public void testGet() throws IOException {
+    public void testCRUD() throws IOException {
         for (int i = 0; i < 5; i++) {
             putModel("se_model_" + i, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING);
         }
@@ -38,7 +41,7 @@ public void testGet() throws IOException {
         }
 
         var getAllModels = getAllModels();
-        int numModels = DefaultElserFeatureFlag.isEnabled() ? 10 : 9;
+        int numModels = DefaultElserFeatureFlag.isEnabled() ? 11 : 9;
         assertThat(getAllModels, hasSize(numModels));
 
         var getSparseModels = getModels("_all", TaskType.SPARSE_EMBEDDING);
@@ -49,15 +52,34 @@ public void testGet() throws IOException {
         }
 
         var getDenseModels = getModels("_all", TaskType.TEXT_EMBEDDING);
-        assertThat(getDenseModels, hasSize(4));
+        int numDenseModels = DefaultElserFeatureFlag.isEnabled() ? 5 : 4;
+        assertThat(getDenseModels, hasSize(numDenseModels));
         for (var denseModel : getDenseModels) {
             assertEquals("text_embedding", denseModel.get("task_type"));
         }
-
-        var singleModel = getModels("se_model_1", TaskType.SPARSE_EMBEDDING);
-        assertThat(singleModel, hasSize(1));
-        assertEquals("se_model_1", singleModel.get(0).get("inference_id"));
-
+        String oldApiKey;
+        {
+            var singleModel = getModels("se_model_1", TaskType.SPARSE_EMBEDDING);
+            assertThat(singleModel, hasSize(1));
+            assertEquals("se_model_1", singleModel.get(0).get("inference_id"));
+            oldApiKey = (String) singleModel.get(0).get("api_key");
+        }
+        var newApiKey = randomAlphaOfLength(10);
+        int temperature = randomIntBetween(1, 10);
+        Map<String, Object> updatedEndpoint = updateEndpoint(
+            "se_model_1",
+            updateConfig(TaskType.SPARSE_EMBEDDING, newApiKey, temperature),
+            TaskType.SPARSE_EMBEDDING
+        );
+        Map<String, Objects> updatedTaskSettings = (Map<String, Objects>) updatedEndpoint.get("task_settings");
+        assertEquals(temperature, updatedTaskSettings.get("temperature"));
+        {
+            var singleModel = getModels("se_model_1", TaskType.SPARSE_EMBEDDING);
+            assertThat(singleModel, hasSize(1));
+            assertEquals("se_model_1", singleModel.get(0).get("inference_id"));
+            assertNotEquals(oldApiKey, newApiKey);
+            assertEquals(updatedEndpoint, singleModel.get(0));
+        }
         for (int i = 0; i < 5; i++) {
             deleteModel("se_model_" + i, TaskType.SPARSE_EMBEDDING);
         }
@@ -305,4 +327,9 @@ public void testSupportedStream() throws Exception {
             deleteModel(modelId);
         }
     }
+
+    public void testGetZeroModels() throws IOException {
+        var models = getModels("_all", TaskType.RERANK);
+        assertThat(models, empty());
+    }
 }
diff --git a/x-pack/plugin/inference/qa/mixed-cluster/build.gradle b/x-pack/plugin/inference/qa/mixed-cluster/build.gradle
index 1d5369468b054..0bc4813f25137 100644
--- a/x-pack/plugin/inference/qa/mixed-cluster/build.gradle
+++ b/x-pack/plugin/inference/qa/mixed-cluster/build.gradle
@@ -20,7 +20,7 @@ dependencies {
 
 // inference is available in 8.11 or later
 def supportedVersion = bwcVersion -> {
-  return bwcVersion.onOrAfter(Version.fromString("8.11.0"));
+  return bwcVersion.onOrAfter(Version.fromString("8.11.0")) && bwcVersion != VersionProperties.elasticsearchVersion
 }
 
 BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName ->
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java
index 02dfff1b5c2e6..6496bcdd89f21 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java
@@ -163,6 +163,11 @@ public String getWriteableName() {
         public TransportVersion getMinimalSupportedVersion() {
             return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests
         }
+
+        @Override
+        public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+            return fromMap(new HashMap<>(newSettings));
+        }
     }
 
     public record TestSecretSettings(String apiKey) implements SecretSettings {
@@ -211,5 +216,10 @@ public String getWriteableName() {
         public TransportVersion getMinimalSupportedVersion() {
             return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests
         }
+
+        @Override
+        public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+            return TestSecretSettings.fromMap(new HashMap<>(newSecrets));
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
index 73c0f6d4c7685..54d83af8f5d95 100644
--- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
+++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
@@ -135,7 +135,7 @@ public void testBulkOperations() throws Exception {
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().size(0).trackTotalHits(true);
         SearchResponse searchResponse = client().search(new SearchRequest(INDEX_NAME).source(sourceBuilder)).get();
         try {
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(totalDocs));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(totalDocs));
         } finally {
             searchResponse.decRef();
         }
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java
index 1a2f0fb6a1137..8713511c2f5f2 100644
--- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java
+++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java
@@ -11,7 +11,11 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.inference.InferenceService;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
@@ -47,6 +51,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -57,6 +62,8 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 
 public class ModelRegistryIT extends ESSingleNodeTestCase {
@@ -122,7 +129,12 @@ public void testGetModel() throws Exception {
         assertEquals(model.getConfigurations().getService(), modelHolder.get().service());
 
         var elserService = new ElasticsearchInternalService(
-            new InferenceServiceExtension.InferenceServiceFactoryContext(mock(Client.class), mock(ThreadPool.class))
+            new InferenceServiceExtension.InferenceServiceFactoryContext(
+                mock(Client.class),
+                mock(ThreadPool.class),
+                mock(ClusterService.class),
+                Settings.EMPTY
+            )
         );
         ElasticsearchInternalModel roundTripModel = (ElasticsearchInternalModel) elserService.parsePersistedConfigWithSecrets(
             modelHolder.get().inferenceEntityId(),
@@ -240,7 +252,7 @@ public void testGetAllModels() throws InterruptedException {
         }
 
         AtomicReference<List<UnparsedModel>> modelHolder = new AtomicReference<>();
-        blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder);
+        blockingCall(listener -> modelRegistry.getAllModels(randomBoolean(), listener), modelHolder, exceptionHolder);
         assertNull(exceptionHolder.get());
         assertThat(modelHolder.get(), hasSize(modelCount));
         var getAllModels = modelHolder.get();
@@ -283,18 +295,30 @@ public void testGetModelWithSecrets() throws InterruptedException {
     }
 
     public void testGetAllModels_WithDefaults() throws Exception {
-        var service = "foo";
-        var secret = "abc";
+        var serviceName = "foo";
         int configuredModelCount = 10;
         int defaultModelCount = 2;
         int totalModelCount = 12;
 
-        var defaultConfigs = new HashMap<String, UnparsedModel>();
+        var service = mock(InferenceService.class);
+
+        var defaultConfigs = new ArrayList<Model>();
+        var defaultIds = new ArrayList<InferenceService.DefaultConfigId>();
         for (int i = 0; i < defaultModelCount; i++) {
             var id = "default-" + i;
-            defaultConfigs.put(id, createUnparsedConfig(id, randomFrom(TaskType.values()), service, secret));
+            var taskType = randomFrom(TaskType.values());
+            defaultConfigs.add(createModel(id, taskType, serviceName));
+            defaultIds.add(new InferenceService.DefaultConfigId(id, taskType, service));
         }
-        defaultConfigs.values().forEach(modelRegistry::addDefaultConfiguration);
+
+        doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            var listener = (ActionListener<List<Model>>) invocation.getArguments()[0];
+            listener.onResponse(defaultConfigs);
+            return Void.TYPE;
+        }).when(service).defaultConfigs(any());
+
+        defaultIds.forEach(modelRegistry::addDefaultIds);
 
         AtomicReference<Boolean> putModelHolder = new AtomicReference<>();
         AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
@@ -302,7 +326,7 @@ public void testGetAllModels_WithDefaults() throws Exception {
         var createdModels = new HashMap<String, Model>();
         for (int i = 0; i < configuredModelCount; i++) {
             var id = randomAlphaOfLength(5) + i;
-            var model = createModel(id, randomFrom(TaskType.values()), service);
+            var model = createModel(id, randomFrom(TaskType.values()), serviceName);
             createdModels.put(id, model);
             blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder);
             assertThat(putModelHolder.get(), is(true));
@@ -310,22 +334,28 @@ public void testGetAllModels_WithDefaults() throws Exception {
         }
 
         AtomicReference<List<UnparsedModel>> modelHolder = new AtomicReference<>();
-        blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder);
+        blockingCall(listener -> modelRegistry.getAllModels(randomBoolean(), listener), modelHolder, exceptionHolder);
         assertNull(exceptionHolder.get());
         assertThat(modelHolder.get(), hasSize(totalModelCount));
         var getAllModels = modelHolder.get();
         assertReturnModelIsModifiable(modelHolder.get().get(0));
 
+        // same result but configs should have been persisted this time
+        blockingCall(listener -> modelRegistry.getAllModels(randomBoolean(), listener), modelHolder, exceptionHolder);
+        assertNull(exceptionHolder.get());
+        assertThat(modelHolder.get(), hasSize(totalModelCount));
+
         // sort in the same order as the returned models
-        var ids = new ArrayList<>(defaultConfigs.keySet().stream().toList());
+        var ids = new ArrayList<>(defaultIds.stream().map(InferenceService.DefaultConfigId::inferenceId).toList());
         ids.addAll(createdModels.keySet().stream().toList());
         ids.sort(String::compareTo);
+        var configsById = defaultConfigs.stream().collect(Collectors.toMap(Model::getInferenceEntityId, Function.identity()));
         for (int i = 0; i < totalModelCount; i++) {
             var id = ids.get(i);
             assertEquals(id, getAllModels.get(i).inferenceEntityId());
             if (id.startsWith("default")) {
-                assertEquals(defaultConfigs.get(id).taskType(), getAllModels.get(i).taskType());
-                assertEquals(defaultConfigs.get(id).service(), getAllModels.get(i).service());
+                assertEquals(configsById.get(id).getTaskType(), getAllModels.get(i).taskType());
+                assertEquals(configsById.get(id).getConfigurations().getService(), getAllModels.get(i).service());
             } else {
                 assertEquals(createdModels.get(id).getTaskType(), getAllModels.get(i).taskType());
                 assertEquals(createdModels.get(id).getConfigurations().getService(), getAllModels.get(i).service());
@@ -334,51 +364,111 @@ public void testGetAllModels_WithDefaults() throws Exception {
     }
 
     public void testGetAllModels_OnlyDefaults() throws Exception {
-        var service = "foo";
-        var secret = "abc";
         int defaultModelCount = 2;
+        var serviceName = "foo";
+        var service = mock(InferenceService.class);
 
-        var defaultConfigs = new HashMap<String, UnparsedModel>();
+        var defaultConfigs = new ArrayList<Model>();
+        var defaultIds = new ArrayList<InferenceService.DefaultConfigId>();
         for (int i = 0; i < defaultModelCount; i++) {
             var id = "default-" + i;
-            defaultConfigs.put(id, createUnparsedConfig(id, randomFrom(TaskType.values()), service, secret));
+            var taskType = randomFrom(TaskType.values());
+            defaultConfigs.add(createModel(id, taskType, serviceName));
+            defaultIds.add(new InferenceService.DefaultConfigId(id, taskType, service));
         }
-        defaultConfigs.values().forEach(modelRegistry::addDefaultConfiguration);
+
+        doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            var listener = (ActionListener<List<Model>>) invocation.getArguments()[0];
+            listener.onResponse(defaultConfigs);
+            return Void.TYPE;
+        }).when(service).defaultConfigs(any());
+
+        defaultIds.forEach(modelRegistry::addDefaultIds);
 
         AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
         AtomicReference<List<UnparsedModel>> modelHolder = new AtomicReference<>();
-        blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder);
+        blockingCall(listener -> modelRegistry.getAllModels(randomBoolean(), listener), modelHolder, exceptionHolder);
         assertNull(exceptionHolder.get());
         assertThat(modelHolder.get(), hasSize(2));
         var getAllModels = modelHolder.get();
         assertReturnModelIsModifiable(modelHolder.get().get(0));
 
         // sort in the same order as the returned models
-        var ids = new ArrayList<>(defaultConfigs.keySet().stream().toList());
+        var configsById = defaultConfigs.stream().collect(Collectors.toMap(Model::getInferenceEntityId, Function.identity()));
+        var ids = new ArrayList<>(configsById.keySet().stream().toList());
         ids.sort(String::compareTo);
         for (int i = 0; i < defaultModelCount; i++) {
             var id = ids.get(i);
             assertEquals(id, getAllModels.get(i).inferenceEntityId());
-            assertEquals(defaultConfigs.get(id).taskType(), getAllModels.get(i).taskType());
-            assertEquals(defaultConfigs.get(id).service(), getAllModels.get(i).service());
+            assertEquals(configsById.get(id).getTaskType(), getAllModels.get(i).taskType());
+            assertEquals(configsById.get(id).getConfigurations().getService(), getAllModels.get(i).service());
         }
     }
 
+    public void testGetAllModels_withDoNotPersist() throws Exception {
+        int defaultModelCount = 2;
+        var serviceName = "foo";
+        var service = mock(InferenceService.class);
+
+        var defaultConfigs = new ArrayList<Model>();
+        var defaultIds = new ArrayList<InferenceService.DefaultConfigId>();
+        for (int i = 0; i < defaultModelCount; i++) {
+            var id = "default-" + i;
+            var taskType = randomFrom(TaskType.values());
+            defaultConfigs.add(createModel(id, taskType, serviceName));
+            defaultIds.add(new InferenceService.DefaultConfigId(id, taskType, service));
+        }
+
+        doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            var listener = (ActionListener<List<Model>>) invocation.getArguments()[0];
+            listener.onResponse(defaultConfigs);
+            return Void.TYPE;
+        }).when(service).defaultConfigs(any());
+
+        defaultIds.forEach(modelRegistry::addDefaultIds);
+
+        AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
+        AtomicReference<List<UnparsedModel>> modelHolder = new AtomicReference<>();
+        blockingCall(listener -> modelRegistry.getAllModels(false, listener), modelHolder, exceptionHolder);
+        assertNull(exceptionHolder.get());
+        assertThat(modelHolder.get(), hasSize(2));
+
+        expectThrows(IndexNotFoundException.class, () -> client().admin().indices().prepareGetIndex().addIndices(".inference").get());
+
+        // this time check the index is created
+        blockingCall(listener -> modelRegistry.getAllModels(true, listener), modelHolder, exceptionHolder);
+        assertNull(exceptionHolder.get());
+        assertThat(modelHolder.get(), hasSize(2));
+        assertInferenceIndexExists();
+    }
+
     public void testGet_WithDefaults() throws InterruptedException {
-        var service = "foo";
-        var secret = "abc";
+        var serviceName = "foo";
+        var service = mock(InferenceService.class);
+
+        var defaultConfigs = new ArrayList<Model>();
+        var defaultIds = new ArrayList<InferenceService.DefaultConfigId>();
 
-        var defaultSparse = createUnparsedConfig("default-sparse", TaskType.SPARSE_EMBEDDING, service, secret);
-        var defaultText = createUnparsedConfig("default-text", TaskType.TEXT_EMBEDDING, service, secret);
+        defaultConfigs.add(createModel("default-sparse", TaskType.SPARSE_EMBEDDING, serviceName));
+        defaultConfigs.add(createModel("default-text", TaskType.TEXT_EMBEDDING, serviceName));
+        defaultIds.add(new InferenceService.DefaultConfigId("default-sparse", TaskType.SPARSE_EMBEDDING, service));
+        defaultIds.add(new InferenceService.DefaultConfigId("default-text", TaskType.TEXT_EMBEDDING, service));
 
-        modelRegistry.addDefaultConfiguration(defaultSparse);
-        modelRegistry.addDefaultConfiguration(defaultText);
+        doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            var listener = (ActionListener<List<Model>>) invocation.getArguments()[0];
+            listener.onResponse(defaultConfigs);
+            return Void.TYPE;
+        }).when(service).defaultConfigs(any());
+        defaultIds.forEach(modelRegistry::addDefaultIds);
 
         AtomicReference<Boolean> putModelHolder = new AtomicReference<>();
         AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
 
-        var configured1 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), service);
-        var configured2 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), service);
+        var configured1 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), serviceName);
+        var configured2 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), serviceName);
         blockingCall(listener -> modelRegistry.storeModel(configured1, listener), putModelHolder, exceptionHolder);
         assertThat(putModelHolder.get(), is(true));
         blockingCall(listener -> modelRegistry.storeModel(configured2, listener), putModelHolder, exceptionHolder);
@@ -387,6 +477,7 @@ public void testGet_WithDefaults() throws InterruptedException {
 
         AtomicReference<UnparsedModel> modelHolder = new AtomicReference<>();
         blockingCall(listener -> modelRegistry.getModel("default-sparse", listener), modelHolder, exceptionHolder);
+        assertNull(exceptionHolder.get());
         assertEquals("default-sparse", modelHolder.get().inferenceEntityId());
         assertEquals(TaskType.SPARSE_EMBEDDING, modelHolder.get().taskType());
         assertReturnModelIsModifiable(modelHolder.get());
@@ -401,23 +492,32 @@ public void testGet_WithDefaults() throws InterruptedException {
     }
 
     public void testGetByTaskType_WithDefaults() throws Exception {
-        var service = "foo";
-        var secret = "abc";
-
-        var defaultSparse = createUnparsedConfig("default-sparse", TaskType.SPARSE_EMBEDDING, service, secret);
-        var defaultText = createUnparsedConfig("default-text", TaskType.TEXT_EMBEDDING, service, secret);
-        var defaultChat = createUnparsedConfig("default-chat", TaskType.COMPLETION, service, secret);
-
-        modelRegistry.addDefaultConfiguration(defaultSparse);
-        modelRegistry.addDefaultConfiguration(defaultText);
-        modelRegistry.addDefaultConfiguration(defaultChat);
+        var serviceName = "foo";
+
+        var defaultSparse = createModel("default-sparse", TaskType.SPARSE_EMBEDDING, serviceName);
+        var defaultText = createModel("default-text", TaskType.TEXT_EMBEDDING, serviceName);
+        var defaultChat = createModel("default-chat", TaskType.COMPLETION, serviceName);
+
+        var service = mock(InferenceService.class);
+        var defaultIds = new ArrayList<InferenceService.DefaultConfigId>();
+        defaultIds.add(new InferenceService.DefaultConfigId("default-sparse", TaskType.SPARSE_EMBEDDING, service));
+        defaultIds.add(new InferenceService.DefaultConfigId("default-text", TaskType.TEXT_EMBEDDING, service));
+        defaultIds.add(new InferenceService.DefaultConfigId("default-chat", TaskType.COMPLETION, service));
+
+        doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            var listener = (ActionListener<List<Model>>) invocation.getArguments()[0];
+            listener.onResponse(List.of(defaultSparse, defaultChat, defaultText));
+            return Void.TYPE;
+        }).when(service).defaultConfigs(any());
+        defaultIds.forEach(modelRegistry::addDefaultIds);
 
         AtomicReference<Boolean> putModelHolder = new AtomicReference<>();
         AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
 
-        var configuredSparse = createModel("configured-sparse", TaskType.SPARSE_EMBEDDING, service);
-        var configuredText = createModel("configured-text", TaskType.TEXT_EMBEDDING, service);
-        var configuredRerank = createModel("configured-rerank", TaskType.RERANK, service);
+        var configuredSparse = createModel("configured-sparse", TaskType.SPARSE_EMBEDDING, serviceName);
+        var configuredText = createModel("configured-text", TaskType.TEXT_EMBEDDING, serviceName);
+        var configuredRerank = createModel("configured-rerank", TaskType.RERANK, serviceName);
         blockingCall(listener -> modelRegistry.storeModel(configuredSparse, listener), putModelHolder, exceptionHolder);
         assertThat(putModelHolder.get(), is(true));
         blockingCall(listener -> modelRegistry.storeModel(configuredText, listener), putModelHolder, exceptionHolder);
@@ -452,6 +552,12 @@ public void testGetByTaskType_WithDefaults() throws Exception {
         assertReturnModelIsModifiable(modelHolder.get().get(0));
     }
 
+    private void assertInferenceIndexExists() {
+        var indexResponse = client().admin().indices().prepareGetIndex().addIndices(".inference").get();
+        assertNotNull(indexResponse.getSettings());
+        assertNotNull(indexResponse.getMappings());
+    }
+
     @SuppressWarnings("unchecked")
     private void assertReturnModelIsModifiable(UnparsedModel unparsedModel) {
         var settings = unparsedModel.settings();
@@ -490,7 +596,6 @@ private Model buildElserModelConfig(String inferenceEntityId, TaskType taskType)
             );
             default -> throw new IllegalArgumentException("task type " + taskType + " is not supported");
         };
-
     }
 
     protected <T> void blockingCall(Consumer<ActionListener<T>> function, AtomicReference<T> response, AtomicReference<Exception> error)
@@ -531,10 +636,6 @@ public static Model createModelWithSecrets(String inferenceEntityId, TaskType ta
         );
     }
 
-    public static UnparsedModel createUnparsedConfig(String inferenceEntityId, TaskType taskType, String service, String secret) {
-        return new UnparsedModel(inferenceEntityId, taskType, service, Map.of("a", "b"), Map.of("secret", secret));
-    }
-
     private static class TestModelOfAnyKind extends ModelConfigurations {
 
         record TestModelServiceSettings() implements ServiceSettings {
@@ -600,6 +701,10 @@ public void writeTo(StreamOutput out) throws IOException {
             public boolean isEmpty() {
                 return true;
             }
+
+            public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+                return this;
+            }
         }
 
         record TestSecretSettings(String key) implements SecretSettings {
@@ -625,6 +730,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.endObject();
                 return builder;
             }
+
+            @Override
+            public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+                return new TestSecretSettings(newSecrets.get("secret").toString());
+            }
         }
 
         TestModelOfAnyKind(String inferenceEntityId, TaskType taskType, String service) {
diff --git a/x-pack/plugin/inference/src/main/java/module-info.java b/x-pack/plugin/inference/src/main/java/module-info.java
index 53cb6ac154ced..60cb254e0afbe 100644
--- a/x-pack/plugin/inference/src/main/java/module-info.java
+++ b/x-pack/plugin/inference/src/main/java/module-info.java
@@ -32,6 +32,7 @@
     requires software.amazon.awssdk.profiles;
     requires org.slf4j;
     requires software.amazon.awssdk.retries.api;
+    requires org.reactivestreams;
 
     exports org.elasticsearch.xpack.inference.action;
     exports org.elasticsearch.xpack.inference.registry;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
index 87b7be717d31b..216b5c984eca5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
@@ -36,4 +36,8 @@ public Set<NodeFeature> getFeatures() {
         return Set.copyOf(features);
     }
 
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(SemanticTextFieldMapper.SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
index 927fd94809886..ebbf1e59e8b1f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
@@ -47,12 +47,14 @@
 import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction;
+import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction;
 import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceEndpointAction;
 import org.elasticsearch.xpack.inference.action.TransportGetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction;
 import org.elasticsearch.xpack.inference.action.TransportInferenceAction;
 import org.elasticsearch.xpack.inference.action.TransportInferenceUsageAction;
 import org.elasticsearch.xpack.inference.action.TransportPutInferenceModelAction;
+import org.elasticsearch.xpack.inference.action.TransportUpdateInferenceModelAction;
 import org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter;
 import org.elasticsearch.xpack.inference.common.Truncator;
 import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockRequestSender;
@@ -76,6 +78,7 @@
 import org.elasticsearch.xpack.inference.rest.RestInferenceAction;
 import org.elasticsearch.xpack.inference.rest.RestPutInferenceModelAction;
 import org.elasticsearch.xpack.inference.rest.RestStreamInferenceAction;
+import org.elasticsearch.xpack.inference.rest.RestUpdateInferenceModelAction;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchService;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockService;
@@ -149,6 +152,7 @@ public InferencePlugin(Settings settings) {
             new ActionHandler<>(InferenceAction.INSTANCE, TransportInferenceAction.class),
             new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class),
             new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class),
+            new ActionHandler<>(UpdateInferenceModelAction.INSTANCE, TransportUpdateInferenceModelAction.class),
             new ActionHandler<>(DeleteInferenceEndpointAction.INSTANCE, TransportDeleteInferenceEndpointAction.class),
             new ActionHandler<>(XPackUsageFeatureAction.INFERENCE, TransportInferenceUsageAction.class),
             new ActionHandler<>(GetInferenceDiagnosticsAction.INSTANCE, TransportGetInferenceDiagnosticsAction.class)
@@ -172,6 +176,7 @@ public List<RestHandler> getRestHandlers(
             new RestStreamInferenceAction(),
             new RestGetInferenceModelAction(),
             new RestPutInferenceModelAction(),
+            new RestUpdateInferenceModelAction(),
             new RestDeleteInferenceEndpointAction(),
             new RestGetInferenceDiagnosticsAction()
         );
@@ -207,13 +212,21 @@ public Collection<?> createComponents(PluginServices services) {
             );
         }
 
-        var factoryContext = new InferenceServiceExtension.InferenceServiceFactoryContext(services.client(), services.threadPool());
+        var factoryContext = new InferenceServiceExtension.InferenceServiceFactoryContext(
+            services.client(),
+            services.threadPool(),
+            services.clusterService(),
+            settings
+        );
+
         // This must be done after the HttpRequestSenderFactory is created so that the services can get the
         // reference correctly
         var registry = new InferenceServiceRegistry(inferenceServices, factoryContext);
         registry.init(services.client());
-        for (var service : registry.getServices().values()) {
-            service.defaultConfigs().forEach(modelRegistry::addDefaultConfiguration);
+        if (DefaultElserFeatureFlag.isEnabled()) {
+            for (var service : registry.getServices().values()) {
+                service.defaultConfigIds().forEach(modelRegistry::addDefaultIds);
+            }
         }
         inferenceServiceRegistry.set(registry);
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
index 829a6b6c67ff9..c1dbd8cfec9d5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
@@ -9,8 +9,6 @@
 
 package org.elasticsearch.xpack.inference.action;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
@@ -47,7 +45,6 @@ public class TransportDeleteInferenceEndpointAction extends TransportMasterNodeA
 
     private final ModelRegistry modelRegistry;
     private final InferenceServiceRegistry serviceRegistry;
-    private static final Logger logger = LogManager.getLogger(TransportDeleteInferenceEndpointAction.class);
     private final Executor executor;
 
     @Inject
@@ -118,7 +115,7 @@ private void doExecuteForked(
 
             var service = serviceRegistry.getService(unparsedModel.service());
             if (service.isPresent()) {
-                service.get().stop(request.getInferenceEndpointId(), listener);
+                service.get().stop(unparsedModel, listener);
             } else {
                 listener.onFailure(
                     new ElasticsearchStatusException(
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java
index 5ee1e40869dbc..01e663df4a3ea 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java
@@ -9,13 +9,13 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.inference.InferenceServiceRegistry;
-import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.injection.guice.Inject;
@@ -29,8 +29,11 @@
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 
 import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.Executor;
+import java.util.stream.Collectors;
 
 public class TransportGetInferenceModelAction extends HandledTransportAction<
     GetInferenceModelAction.Request,
@@ -69,7 +72,7 @@ protected void doExecute(
         boolean inferenceEntityIdIsWildCard = Strings.isAllOrWildcard(request.getInferenceEntityId());
 
         if (request.getTaskType() == TaskType.ANY && inferenceEntityIdIsWildCard) {
-            getAllModels(listener);
+            getAllModels(request.isPersistDefaultConfig(), listener);
         } else if (inferenceEntityIdIsWildCard) {
             getModelsByTaskType(request.getTaskType(), listener);
         } else {
@@ -96,38 +99,77 @@ private void getSingleModel(
 
             var model = service.get()
                 .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings());
-            delegate.onResponse(new GetInferenceModelAction.Response(List.of(model.getConfigurations())));
+
+            service.get()
+                .updateModelsWithDynamicFields(
+                    List.of(model),
+                    delegate.delegateFailureAndWrap(
+                        (l2, updatedModels) -> l2.onResponse(
+                            new GetInferenceModelAction.Response(
+                                updatedModels.stream().map(Model::getConfigurations).collect(Collectors.toList())
+                            )
+                        )
+                    )
+                );
         }));
     }
 
-    private void getAllModels(ActionListener<GetInferenceModelAction.Response> listener) {
+    private void getAllModels(boolean persistDefaultEndpoints, ActionListener<GetInferenceModelAction.Response> listener) {
         modelRegistry.getAllModels(
-            listener.delegateFailureAndWrap((l, models) -> executor.execute(ActionRunnable.supply(l, () -> parseModels(models))))
+            persistDefaultEndpoints,
+            listener.delegateFailureAndWrap((l, models) -> executor.execute(() -> parseModels(models, listener)))
         );
     }
 
     private void getModelsByTaskType(TaskType taskType, ActionListener<GetInferenceModelAction.Response> listener) {
         modelRegistry.getModelsByTaskType(
             taskType,
-            listener.delegateFailureAndWrap((l, models) -> executor.execute(ActionRunnable.supply(l, () -> parseModels(models))))
+            listener.delegateFailureAndWrap((l, models) -> executor.execute(() -> parseModels(models, listener)))
         );
     }
 
-    private GetInferenceModelAction.Response parseModels(List<UnparsedModel> unparsedModels) {
-        var parsedModels = new ArrayList<ModelConfigurations>();
+    private void parseModels(List<UnparsedModel> unparsedModels, ActionListener<GetInferenceModelAction.Response> listener) {
+        if (unparsedModels.isEmpty()) {
+            listener.onResponse(new GetInferenceModelAction.Response(List.of()));
+            return;
+        }
 
-        for (var unparsedModel : unparsedModels) {
-            var service = serviceRegistry.getService(unparsedModel.service());
-            if (service.isEmpty()) {
-                throw serviceNotFoundException(unparsedModel.service(), unparsedModel.inferenceEntityId());
+        var parsedModelsByService = new HashMap<String, List<Model>>();
+        try {
+            for (var unparsedModel : unparsedModels) {
+                var service = serviceRegistry.getService(unparsedModel.service());
+                if (service.isEmpty()) {
+                    throw serviceNotFoundException(unparsedModel.service(), unparsedModel.inferenceEntityId());
+                }
+                var list = parsedModelsByService.computeIfAbsent(service.get().name(), s -> new ArrayList<>());
+                list.add(
+                    service.get()
+                        .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings())
+                );
             }
-            parsedModels.add(
-                service.get()
-                    .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings())
-                    .getConfigurations()
+
+            var groupedListener = new GroupedActionListener<List<Model>>(
+                parsedModelsByService.entrySet().size(),
+                listener.delegateFailureAndWrap((delegate, listOfListOfModels) -> {
+                    var modifiable = new ArrayList<Model>();
+                    for (var l : listOfListOfModels) {
+                        modifiable.addAll(l);
+                    }
+                    modifiable.sort(Comparator.comparing(Model::getInferenceEntityId));
+                    delegate.onResponse(
+                        new GetInferenceModelAction.Response(modifiable.stream().map(Model::getConfigurations).collect(Collectors.toList()))
+                    );
+                })
             );
+
+            for (var entry : parsedModelsByService.entrySet()) {
+                serviceRegistry.getService(entry.getKey())
+                    .get()  // must be non-null to get this far
+                    .updateModelsWithDynamicFields(entry.getValue(), groupedListener);
+            }
+        } catch (Exception e) {
+            listener.onFailure(e);
         }
-        return new GetInferenceModelAction.Response(parsedModels);
     }
 
     private ElasticsearchStatusException serviceNotFoundException(String service, String inferenceId) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java
index 624afff9f5d11..7b7475efac334 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java
@@ -63,7 +63,7 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<XPackUsageFeatureResponse> listener
     ) {
-        GetInferenceModelAction.Request getInferenceModelAction = new GetInferenceModelAction.Request("_all", TaskType.ANY);
+        GetInferenceModelAction.Request getInferenceModelAction = new GetInferenceModelAction.Request("_all", TaskType.ANY, false);
         client.execute(GetInferenceModelAction.INSTANCE, getInferenceModelAction, listener.delegateFailureAndWrap((delegate, response) -> {
             Map<String, InferenceFeatureSetUsage.ModelStats> stats = new TreeMap<>();
             for (ModelConfigurations model : response.getEndpoints()) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
index 49d65b6e0dc59..64eeed82ee1b9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.inference.InferencePlugin;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService;
 
 import java.io.IOException;
@@ -100,7 +101,7 @@ protected void masterOperation(
         ActionListener<PutInferenceModelAction.Response> listener
     ) throws Exception {
         var requestAsMap = requestToMap(request);
-        var resolvedTaskType = resolveTaskType(request.getTaskType(), (String) requestAsMap.remove(TaskType.NAME));
+        var resolvedTaskType = ServiceUtils.resolveTaskType(request.getTaskType(), (String) requestAsMap.remove(TaskType.NAME));
 
         String serviceName = (String) requestAsMap.remove(ModelConfigurations.SERVICE);
         if (serviceName == null) {
@@ -227,37 +228,4 @@ protected ClusterBlockException checkBlock(PutInferenceModelAction.Request reque
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
     }
 
-    /**
-     * task_type can be specified as either a URL parameter or in the
-     * request body. Resolve which to use or throw if the settings are
-     * inconsistent
-     * @param urlTaskType Taken from the URL parameter. ANY means not specified.
-     * @param bodyTaskType Taken from the request body. Maybe null
-     * @return The resolved task type
-     */
-    static TaskType resolveTaskType(TaskType urlTaskType, String bodyTaskType) {
-        if (bodyTaskType == null) {
-            if (urlTaskType == TaskType.ANY) {
-                throw new ElasticsearchStatusException("model is missing required setting [task_type]", RestStatus.BAD_REQUEST);
-            } else {
-                return urlTaskType;
-            }
-        }
-
-        TaskType parsedBodyTask = TaskType.fromStringOrStatusException(bodyTaskType);
-        if (parsedBodyTask == TaskType.ANY) {
-            throw new ElasticsearchStatusException("task_type [any] is not valid type for inference", RestStatus.BAD_REQUEST);
-        }
-
-        if (parsedBodyTask.isAnyOrSame(urlTaskType) == false) {
-            throw new ElasticsearchStatusException(
-                "Cannot resolve conflicting task_type parameter in the request URL [{}] and the request body [{}]",
-                RestStatus.BAD_REQUEST,
-                urlTaskType.toString(),
-                bodyTaskType
-            );
-        }
-
-        return parsedBodyTask;
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java
new file mode 100644
index 0000000000000..03a88e5228fa8
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java
@@ -0,0 +1,328 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.inference.InferenceService;
+import org.elasticsearch.inference.InferenceServiceRegistry;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SecretSettings;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.inference.UnparsedModel;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction;
+import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction;
+import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentUtils;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.inference.registry.ModelRegistry;
+import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService;
+import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.resolveTaskType;
+import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS;
+
+public class TransportUpdateInferenceModelAction extends TransportMasterNodeAction<
+    UpdateInferenceModelAction.Request,
+    UpdateInferenceModelAction.Response> {
+
+    private static final Logger logger = LogManager.getLogger(TransportUpdateInferenceModelAction.class);
+
+    private final ModelRegistry modelRegistry;
+    private final InferenceServiceRegistry serviceRegistry;
+    private final Client client;
+
+    @Inject
+    public TransportUpdateInferenceModelAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        ModelRegistry modelRegistry,
+        InferenceServiceRegistry serviceRegistry,
+        Client client,
+        Settings settings
+    ) {
+        super(
+            UpdateInferenceModelAction.NAME,
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            UpdateInferenceModelAction.Request::new,
+            indexNameExpressionResolver,
+            UpdateInferenceModelAction.Response::new,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.modelRegistry = modelRegistry;
+        this.serviceRegistry = serviceRegistry;
+        this.client = client;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        UpdateInferenceModelAction.Request request,
+        ClusterState state,
+        ActionListener<UpdateInferenceModelAction.Response> masterListener
+    ) {
+        var bodyTaskType = request.getContentAsSettings().taskType();
+        var resolvedTaskType = resolveTaskType(request.getTaskType(), bodyTaskType != null ? bodyTaskType.toString() : null);
+
+        AtomicReference<InferenceService> service = new AtomicReference<>();
+
+        var inferenceEntityId = request.getInferenceEntityId();
+
+        SubscribableListener.<UnparsedModel>newForked(listener -> { checkEndpointExists(inferenceEntityId, listener); })
+            .<UnparsedModel>andThen((listener, unparsedModel) -> {
+
+                Optional<InferenceService> optionalService = serviceRegistry.getService(unparsedModel.service());
+                if (optionalService.isEmpty()) {
+                    listener.onFailure(
+                        new ElasticsearchStatusException(
+                            "Service [{}] not found",
+                            RestStatus.INTERNAL_SERVER_ERROR,
+                            unparsedModel.service()
+                        )
+                    );
+                } else {
+                    service.set(optionalService.get());
+                    listener.onResponse(unparsedModel);
+                }
+            })
+            .<Boolean>andThen((listener, existingUnparsedModel) -> {
+
+                Model existingParsedModel = service.get()
+                    .parsePersistedConfigWithSecrets(
+                        request.getInferenceEntityId(),
+                        existingUnparsedModel.taskType(),
+                        new HashMap<>(existingUnparsedModel.settings()),
+                        new HashMap<>(existingUnparsedModel.secrets())
+                    );
+
+                Model newModel = combineExistingModelWithNewSettings(
+                    existingParsedModel,
+                    request.getContentAsSettings(),
+                    service.get().name(),
+                    resolvedTaskType
+                );
+
+                if (isInClusterService(service.get().name())) {
+                    updateInClusterEndpoint(request, newModel, existingParsedModel, listener);
+                } else {
+                    modelRegistry.updateModelTransaction(newModel, existingParsedModel, listener);
+                }
+            })
+            .<ModelConfigurations>andThen((listener, didUpdate) -> {
+                if (didUpdate) {
+                    modelRegistry.getModel(inferenceEntityId, ActionListener.wrap((unparsedModel) -> {
+                        if (unparsedModel == null) {
+                            listener.onFailure(
+                                new ElasticsearchStatusException(
+                                    "Failed to update model, updated model not found",
+                                    RestStatus.INTERNAL_SERVER_ERROR
+                                )
+                            );
+                        } else {
+                            listener.onResponse(
+                                service.get()
+                                    .parsePersistedConfig(
+                                        request.getInferenceEntityId(),
+                                        resolvedTaskType,
+                                        new HashMap<>(unparsedModel.settings())
+                                    )
+                                    .getConfigurations()
+                            );
+                        }
+                    }, listener::onFailure));
+                } else {
+                    listener.onFailure(new ElasticsearchStatusException("Failed to update model", RestStatus.INTERNAL_SERVER_ERROR));
+                }
+
+            }).<UpdateInferenceModelAction.Response>andThen((listener, modelConfig) -> {
+                listener.onResponse(new UpdateInferenceModelAction.Response(modelConfig));
+            })
+            .addListener(masterListener);
+    }
+
+    /**
+     * Combines the existing model with the new settings to create a new model using the
+     * SecretSettings and TaskSettings implementations for each service, as well as specifically handling NUM_ALLOCATIONS.
+     *
+     * @param existingParsedModel the Model representing a third-party service endpoint
+     * @param settingsToUpdate    new settings
+     * @param serviceName
+     * @return a new object representing the updated model
+     */
+    private Model combineExistingModelWithNewSettings(
+        Model existingParsedModel,
+        UpdateInferenceModelAction.Settings settingsToUpdate,
+        String serviceName,
+        TaskType resolvedTaskType
+    ) {
+        ModelConfigurations existingConfigs = existingParsedModel.getConfigurations();
+        TaskSettings existingTaskSettings = existingConfigs.getTaskSettings();
+        SecretSettings existingSecretSettings = existingParsedModel.getSecretSettings();
+
+        SecretSettings newSecretSettings = existingSecretSettings;
+        TaskSettings newTaskSettings = existingTaskSettings;
+        ServiceSettings newServiceSettings = existingConfigs.getServiceSettings();
+
+        if (settingsToUpdate.serviceSettings() != null && existingSecretSettings != null) {
+            newSecretSettings = existingSecretSettings.newSecretSettings(settingsToUpdate.serviceSettings());
+        }
+        if (settingsToUpdate.serviceSettings() != null && settingsToUpdate.serviceSettings().containsKey(NUM_ALLOCATIONS)) {
+            // In cluster services can only have their num_allocations updated, so this is a special case
+            if (newServiceSettings instanceof ElasticsearchInternalServiceSettings elasticServiceSettings) {
+                newServiceSettings = new ElasticsearchInternalServiceSettings(
+                    elasticServiceSettings,
+                    (Integer) settingsToUpdate.serviceSettings().get(NUM_ALLOCATIONS)
+                );
+            }
+        }
+        if (settingsToUpdate.taskSettings() != null && existingTaskSettings != null) {
+            newTaskSettings = existingTaskSettings.updatedTaskSettings(settingsToUpdate.taskSettings());
+        }
+
+        if (existingParsedModel.getTaskType().equals(resolvedTaskType) == false) {
+            throw new ElasticsearchStatusException("Task type must match the task type of the existing endpoint", RestStatus.BAD_REQUEST);
+        }
+
+        ModelConfigurations newModelConfigs = new ModelConfigurations(
+            existingParsedModel.getInferenceEntityId(),
+            existingParsedModel.getTaskType(),
+            serviceName,
+            newServiceSettings,
+            newTaskSettings
+        );
+
+        return new Model(newModelConfigs, new ModelSecrets(newSecretSettings));
+    }
+
+    private void updateInClusterEndpoint(
+        UpdateInferenceModelAction.Request request,
+        Model newModel,
+        Model existingParsedModel,
+        ActionListener<Boolean> listener
+    ) throws IOException {
+        // The model we are trying to update must have a trained model associated with it if it is an in-cluster deployment
+        throwIfTrainedModelDoesntExist(request);
+
+        Map<String, Object> serviceSettings = request.getContentAsSettings().serviceSettings();
+        if (serviceSettings != null && serviceSettings.get(NUM_ALLOCATIONS) instanceof Integer numAllocations) {
+
+            UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(
+                request.getInferenceEntityId()
+            );
+            updateRequest.setNumberOfAllocations(numAllocations);
+
+            var delegate = listener.<CreateTrainedModelAssignmentAction.Response>delegateFailure((l2, response) -> {
+                modelRegistry.updateModelTransaction(newModel, existingParsedModel, l2);
+            });
+
+            logger.info(
+                "Updating trained model deployment for inference entity [{}] with [{}] num_allocations",
+                request.getInferenceEntityId(),
+                numAllocations
+            );
+            client.execute(UpdateTrainedModelDeploymentAction.INSTANCE, updateRequest, delegate);
+
+        } else {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Failed to parse [{}] of update request [{}]",
+                    RestStatus.BAD_REQUEST,
+                    NUM_ALLOCATIONS,
+                    request.getContent().utf8ToString()
+                )
+            );
+        }
+
+    }
+
+    private boolean isInClusterService(String name) {
+        return List.of(ElasticsearchInternalService.NAME, ElasticsearchInternalService.OLD_ELSER_SERVICE_NAME).contains(name);
+    }
+
+    private void throwIfTrainedModelDoesntExist(UpdateInferenceModelAction.Request request) throws ElasticsearchStatusException {
+        var assignments = TrainedModelAssignmentUtils.modelAssignments(request.getInferenceEntityId(), clusterService.state());
+        if ((assignments == null || assignments.isEmpty())) {
+            throw ExceptionsHelper.entityNotFoundException(
+                Messages.MODEL_ID_DOES_NOT_MATCH_EXISTING_MODEL_IDS_BUT_MUST_FOR_IN_CLUSTER_SERVICE,
+                request.getInferenceEntityId()
+
+            );
+        }
+    }
+
+    private void checkEndpointExists(String inferenceEntityId, ActionListener<UnparsedModel> listener) {
+        modelRegistry.getModelWithSecrets(inferenceEntityId, ActionListener.wrap((model) -> {
+            if (model == null) {
+                listener.onFailure(
+                    ExceptionsHelper.entityNotFoundException(Messages.INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE, inferenceEntityId)
+                );
+            } else {
+                listener.onResponse(model);
+            }
+        }, e -> {
+            if (e instanceof ResourceNotFoundException) {
+                listener.onFailure(
+                    // provide a more specific error message if the inference entity does not exist
+                    ExceptionsHelper.entityNotFoundException(Messages.INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE, inferenceEntityId)
+                );
+            } else {
+                listener.onFailure(e);
+            }
+        }));
+    }
+
+    private static XContentParser getParser(UpdateInferenceModelAction.Request request) throws IOException {
+        return XContentHelper.createParser(XContentParserConfiguration.EMPTY, request.getContent(), request.getContentType());
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(UpdateInferenceModelAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
index a4eb94c2674d1..1a6e4760fe125 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
@@ -397,7 +397,7 @@ private void applyInferenceResponses(BulkItemRequest item, FieldInferenceRespons
                     ),
                     indexRequest.getContentType()
                 );
-                newDocMap.put(fieldName, result);
+                SemanticTextFieldMapper.insertValue(fieldName, newDocMap, result);
             }
             indexRequest.source(newDocMap, indexRequest.getContentType());
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java
index 477c3ea6352f5..20520ca829297 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java
@@ -13,7 +13,7 @@
 import java.util.Map;
 
 public class ChunkingSettingsBuilder {
-    public static final WordBoundaryChunkingSettings DEFAULT_SETTINGS = new WordBoundaryChunkingSettings(250, 100);
+    public static final SentenceBoundaryChunkingSettings DEFAULT_SETTINGS = new SentenceBoundaryChunkingSettings(250, 1);
 
     public static ChunkingSettings fromMap(Map<String, Object> settings) {
         if (settings.isEmpty()) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java
index 3ae8dc0550391..c5897f32d6eb8 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java
@@ -324,7 +324,7 @@ private ElasticsearchStatusException unexpectedResultTypeException(String got, S
         public void onFailure(Exception e) {
             var errorResult = new ErrorChunkedInferenceResults(e);
             for (var pos : positions) {
-                errors.setOnce(pos.inputIndex(), errorResult);
+                errors.set(pos.inputIndex(), errorResult);
             }
 
             if (resultCount.incrementAndGet() == totalNumberOfRequests) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java
index 758dd5d04e268..def52e97666f9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java
@@ -29,13 +29,15 @@
 public class SentenceBoundaryChunkingSettings implements ChunkingSettings {
     public static final String NAME = "SentenceBoundaryChunkingSettings";
     private static final ChunkingStrategy STRATEGY = ChunkingStrategy.SENTENCE;
+    private static final int MAX_CHUNK_SIZE_LOWER_LIMIT = 20;
+    private static final int MAX_CHUNK_SIZE_UPPER_LIMIT = 300;
     private static final Set<String> VALID_KEYS = Set.of(
         ChunkingSettingsOptions.STRATEGY.toString(),
         ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(),
         ChunkingSettingsOptions.SENTENCE_OVERLAP.toString()
     );
 
-    private static int DEFAULT_OVERLAP = 0;
+    private static int DEFAULT_OVERLAP = 1;
 
     protected final int maxChunkSize;
     protected int sentenceOverlap = DEFAULT_OVERLAP;
@@ -62,24 +64,27 @@ public static SentenceBoundaryChunkingSettings fromMap(Map<String, Object> map)
             );
         }
 
-        Integer maxChunkSize = ServiceUtils.extractRequiredPositiveInteger(
+        Integer maxChunkSize = ServiceUtils.extractRequiredPositiveIntegerBetween(
             map,
             ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(),
+            MAX_CHUNK_SIZE_LOWER_LIMIT,
+            MAX_CHUNK_SIZE_UPPER_LIMIT,
             ModelConfigurations.CHUNKING_SETTINGS,
             validationException
         );
 
-        Integer sentenceOverlap = ServiceUtils.extractOptionalPositiveInteger(
+        Integer sentenceOverlap = ServiceUtils.removeAsType(
             map,
             ChunkingSettingsOptions.SENTENCE_OVERLAP.toString(),
-            ModelConfigurations.CHUNKING_SETTINGS,
+            Integer.class,
             validationException
         );
-
-        if (sentenceOverlap != null && sentenceOverlap > 1) {
+        if (sentenceOverlap == null) {
+            sentenceOverlap = DEFAULT_OVERLAP;
+        } else if (sentenceOverlap > 1 || sentenceOverlap < 0) {
             validationException.addValidationError(
-                ChunkingSettingsOptions.SENTENCE_OVERLAP.toString() + "[" + sentenceOverlap + "] must be either 0 or 1"
-            ); // todo better
+                ChunkingSettingsOptions.SENTENCE_OVERLAP + "[" + sentenceOverlap + "] must be either 0 or 1"
+            );
         }
 
         if (validationException.validationErrors().isEmpty() == false) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java
index 5b91e122b9c80..7fb0fdc91bf72 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java
@@ -28,6 +28,8 @@
 public class WordBoundaryChunkingSettings implements ChunkingSettings {
     public static final String NAME = "WordBoundaryChunkingSettings";
     private static final ChunkingStrategy STRATEGY = ChunkingStrategy.WORD;
+    private static final int MAX_CHUNK_SIZE_LOWER_LIMIT = 10;
+    private static final int MAX_CHUNK_SIZE_UPPER_LIMIT = 300;
     private static final Set<String> VALID_KEYS = Set.of(
         ChunkingSettingsOptions.STRATEGY.toString(),
         ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(),
@@ -56,9 +58,11 @@ public static WordBoundaryChunkingSettings fromMap(Map<String, Object> map) {
             );
         }
 
-        Integer maxChunkSize = ServiceUtils.extractRequiredPositiveInteger(
+        Integer maxChunkSize = ServiceUtils.extractRequiredPositiveIntegerBetween(
             map,
             ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(),
+            MAX_CHUNK_SIZE_LOWER_LIMIT,
+            MAX_CHUNK_SIZE_UPPER_LIMIT,
             ModelConfigurations.CHUNKING_SETTINGS,
             validationException
         );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java
deleted file mode 100644
index 3871b5fb98882..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.action.googleaistudio;
-
-import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
-import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction;
-import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction;
-import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager;
-import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager;
-import org.elasticsearch.xpack.inference.external.http.sender.Sender;
-import org.elasticsearch.xpack.inference.services.ServiceComponents;
-import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
-import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
-
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
-
-public class GoogleAiStudioActionCreator implements GoogleAiStudioActionVisitor {
-
-    private static final String COMPLETION_ERROR_MESSAGE = "Google AI Studio completion";
-    private final Sender sender;
-
-    private final ServiceComponents serviceComponents;
-
-    public GoogleAiStudioActionCreator(Sender sender, ServiceComponents serviceComponents) {
-        this.sender = Objects.requireNonNull(sender);
-        this.serviceComponents = Objects.requireNonNull(serviceComponents);
-    }
-
-    @Override
-    public ExecutableAction create(GoogleAiStudioCompletionModel model, Map<String, Object> taskSettings) {
-        // no overridden model as task settings are always empty for Google AI Studio completion model
-        var requestManager = new GoogleAiStudioCompletionRequestManager(model, serviceComponents.threadPool());
-        var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), COMPLETION_ERROR_MESSAGE);
-        return new SingleInputSenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage, COMPLETION_ERROR_MESSAGE);
-    }
-
-    @Override
-    public ExecutableAction create(GoogleAiStudioEmbeddingsModel model, Map<String, Object> taskSettings) {
-        var requestManager = new GoogleAiStudioEmbeddingsRequestManager(
-            model,
-            serviceComponents.truncator(),
-            serviceComponents.threadPool()
-        );
-        var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio embeddings");
-        return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage);
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java
deleted file mode 100644
index 2e89200cce53b..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.action.googleaistudio;
-
-import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
-import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
-import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
-
-import java.util.Map;
-
-public interface GoogleAiStudioActionVisitor {
-
-    ExecutableAction create(GoogleAiStudioCompletionModel model, Map<String, Object> taskSettings);
-
-    ExecutableAction create(GoogleAiStudioEmbeddingsModel model, Map<String, Object> taskSettings);
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java
index a4e0c399517c1..2afa91d4dc776 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
 import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockChatCompletionRequest;
 import org.elasticsearch.xpack.inference.external.response.amazonbedrock.AmazonBedrockResponseHandler;
 import org.elasticsearch.xpack.inference.external.response.amazonbedrock.completion.AmazonBedrockChatCompletionResponseListener;
@@ -33,11 +34,16 @@ protected AmazonBedrockChatCompletionExecutor(
 
     @Override
     protected void executeClientRequest(AmazonBedrockBaseClient awsBedrockClient) {
-        var chatCompletionResponseListener = new AmazonBedrockChatCompletionResponseListener(
-            chatCompletionRequest,
-            responseHandler,
-            inferenceResultsListener
-        );
-        chatCompletionRequest.executeChatCompletionRequest(awsBedrockClient, chatCompletionResponseListener);
+        if (chatCompletionRequest.isStreaming()) {
+            var publisher = chatCompletionRequest.executeStreamChatCompletionRequest(awsBedrockClient);
+            inferenceResultsListener.onResponse(new StreamingChatCompletionResults(publisher));
+        } else {
+            var chatCompletionResponseListener = new AmazonBedrockChatCompletionResponseListener(
+                chatCompletionRequest,
+                responseHandler,
+                inferenceResultsListener
+            );
+            chatCompletionRequest.executeChatCompletionRequest(awsBedrockClient, chatCompletionResponseListener);
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java
index 23b6884ddc33a..f1cfc84643b1c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java
@@ -9,17 +9,22 @@
 
 import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
 import software.amazon.awssdk.services.bedrockruntime.model.ConverseResponse;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest;
 import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelRequest;
 import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 
 import java.time.Instant;
+import java.util.concurrent.Flow;
 
 public interface AmazonBedrockClient {
     void converse(ConverseRequest converseRequest, ActionListener<ConverseResponse> responseListener) throws ElasticsearchException;
 
+    Flow.Publisher<? extends ChunkedToXContent> converseStream(ConverseStreamRequest converseStreamRequest) throws ElasticsearchException;
+
     void invokeModel(InvokeModelRequest invokeModelRequest, ActionListener<InvokeModelResponse> responseListener)
         throws ElasticsearchException;
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java
index b1486f4995b84..040aa99d81346 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java
@@ -17,16 +17,21 @@
 import software.amazon.awssdk.services.bedrockruntime.model.BedrockRuntimeException;
 import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
 import software.amazon.awssdk.services.bedrockruntime.model.ConverseResponse;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler;
 import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelRequest;
 import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockModel;
+import org.reactivestreams.FlowAdapters;
 import org.slf4j.LoggerFactory;
 
 import java.security.AccessController;
@@ -36,6 +41,7 @@
 import java.util.Objects;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Flow;
 
 /**
  * Not marking this as "final" so we can subclass it for mocking
@@ -53,19 +59,21 @@ public class AmazonBedrockInferenceClient extends AmazonBedrockBaseClient {
     private static final Duration DEFAULT_CLIENT_TIMEOUT_MS = Duration.ofMillis(10000);
 
     private final BedrockRuntimeAsyncClient internalClient;
+    private final ThreadPool threadPool;
     private volatile Instant expiryTimestamp;
 
-    public static AmazonBedrockBaseClient create(AmazonBedrockModel model, @Nullable TimeValue timeout) {
+    public static AmazonBedrockBaseClient create(AmazonBedrockModel model, @Nullable TimeValue timeout, ThreadPool threadPool) {
         try {
-            return new AmazonBedrockInferenceClient(model, timeout);
+            return new AmazonBedrockInferenceClient(model, timeout, threadPool);
         } catch (Exception e) {
             throw new ElasticsearchException("Failed to create Amazon Bedrock Client", e);
         }
     }
 
-    protected AmazonBedrockInferenceClient(AmazonBedrockModel model, @Nullable TimeValue timeout) {
+    protected AmazonBedrockInferenceClient(AmazonBedrockModel model, @Nullable TimeValue timeout, ThreadPool threadPool) {
         super(model, timeout);
         this.internalClient = createAmazonBedrockClient(model, timeout);
+        this.threadPool = Objects.requireNonNull(threadPool);
         setExpiryTimestamp();
     }
 
@@ -79,6 +87,16 @@ public void converse(ConverseRequest converseRequest, ActionListener<ConverseRes
         }
     }
 
+    @Override
+    public Flow.Publisher<? extends ChunkedToXContent> converseStream(ConverseStreamRequest request) throws ElasticsearchException {
+        var awsResponseProcessor = new AmazonBedrockStreamingChatProcessor(threadPool);
+        internalClient.converseStream(
+            request,
+            ConverseStreamResponseHandler.builder().subscriber(() -> FlowAdapters.toSubscriber(awsResponseProcessor)).build()
+        );
+        return awsResponseProcessor;
+    }
+
     private void onFailure(ActionListener<?> listener, Throwable t, String method) {
         var unwrappedException = t;
         if (t instanceof CompletionException || t instanceof ExecutionException) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java
index 21e5cfaf211e5..339673e1302ac 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java
@@ -29,12 +29,9 @@ public final class AmazonBedrockInferenceClientCache implements AmazonBedrockCli
     // not final for testing
     private Clock clock;
 
-    public AmazonBedrockInferenceClientCache(
-        BiFunction<AmazonBedrockModel, TimeValue, AmazonBedrockBaseClient> creator,
-        @Nullable Clock clock
-    ) {
+    public AmazonBedrockInferenceClientCache(BiFunction<AmazonBedrockModel, TimeValue, AmazonBedrockBaseClient> creator, Clock clock) {
         this.creator = Objects.requireNonNull(creator);
-        this.clock = Objects.requireNonNullElse(clock, Clock.systemUTC());
+        this.clock = Objects.requireNonNull(clock);
     }
 
     public AmazonBedrockBaseClient getOrCreateClient(AmazonBedrockModel model, @Nullable TimeValue timeout) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java
index e23b0274ede26..a8d85d896d684 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 
 import java.io.IOException;
+import java.time.Clock;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -42,7 +43,10 @@ public Factory(ServiceComponents serviceComponents, ClusterService clusterServic
         }
 
         public Sender createSender() {
-            var clientCache = new AmazonBedrockInferenceClientCache(AmazonBedrockInferenceClient::create, null);
+            var clientCache = new AmazonBedrockInferenceClientCache(
+                (model, timeout) -> AmazonBedrockInferenceClient.create(model, timeout, serviceComponents.threadPool()),
+                Clock.systemUTC()
+            );
             return createSender(new AmazonBedrockExecuteOnlyRequestSender(clientCache, serviceComponents.throttlerManager()));
         }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessor.java
new file mode 100644
index 0000000000000..439fc5b65efd5
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessor.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.amazonbedrock;
+
+import software.amazon.awssdk.services.bedrockruntime.model.ContentBlockDeltaEvent;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamOutput;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
+
+import java.util.ArrayDeque;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Flow;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME;
+
+class AmazonBedrockStreamingChatProcessor implements Flow.Processor<ConverseStreamOutput, StreamingChatCompletionResults.Results> {
+    private final AtomicReference<Throwable> error = new AtomicReference<>(null);
+    private final AtomicLong demand = new AtomicLong(0);
+    private final AtomicBoolean isDone = new AtomicBoolean(false);
+    private final AtomicBoolean onCompleteCalled = new AtomicBoolean(false);
+    private final AtomicBoolean onErrorCalled = new AtomicBoolean(false);
+    private final ThreadPool threadPool;
+    private volatile Flow.Subscriber<? super StreamingChatCompletionResults.Results> downstream;
+    private volatile Flow.Subscription upstream;
+
+    AmazonBedrockStreamingChatProcessor(ThreadPool threadPool) {
+        this.threadPool = threadPool;
+    }
+
+    @Override
+    public void subscribe(Flow.Subscriber<? super StreamingChatCompletionResults.Results> subscriber) {
+        if (downstream == null) {
+            downstream = subscriber;
+            downstream.onSubscribe(new StreamSubscription());
+        } else {
+            subscriber.onError(new IllegalStateException("Subscriber already set."));
+        }
+    }
+
+    @Override
+    public void onSubscribe(Flow.Subscription subscription) {
+        if (upstream == null) {
+            upstream = subscription;
+            var currentRequestCount = demand.getAndUpdate(i -> 0);
+            if (currentRequestCount > 0) {
+                upstream.request(currentRequestCount);
+            }
+        } else {
+            subscription.cancel();
+        }
+    }
+
+    @Override
+    public void onNext(ConverseStreamOutput item) {
+        if (item.sdkEventType() == ConverseStreamOutput.EventType.CONTENT_BLOCK_DELTA) {
+            demand.set(0); // reset demand before we fork to another thread
+            item.accept(ConverseStreamResponseHandler.Visitor.builder().onContentBlockDelta(this::sendDownstreamOnAnotherThread).build());
+        } else {
+            upstream.request(1);
+        }
+    }
+
+    // this is always called from a netty thread maintained by the AWS SDK, we'll move it to our thread to process the response
+    private void sendDownstreamOnAnotherThread(ContentBlockDeltaEvent event) {
+        CompletableFuture.runAsync(() -> {
+            var text = event.delta().text();
+            var result = new ArrayDeque<StreamingChatCompletionResults.Result>(1);
+            result.offer(new StreamingChatCompletionResults.Result(text));
+            var results = new StreamingChatCompletionResults.Results(result);
+            downstream.onNext(results);
+        }, threadPool.executor(UTILITY_THREAD_POOL_NAME));
+    }
+
+    @Override
+    public void onError(Throwable amazonBedrockRuntimeException) {
+        error.set(
+            new ElasticsearchException(
+                Strings.format("AmazonBedrock StreamingChatProcessor failure: [%s]", amazonBedrockRuntimeException.getMessage()),
+                amazonBedrockRuntimeException
+            )
+        );
+        if (isDone.compareAndSet(false, true) && checkAndResetDemand() && onErrorCalled.compareAndSet(false, true)) {
+            downstream.onError(error.get());
+        }
+    }
+
+    private boolean checkAndResetDemand() {
+        return demand.getAndUpdate(i -> 0L) > 0L;
+    }
+
+    @Override
+    public void onComplete() {
+        if (isDone.compareAndSet(false, true) && checkAndResetDemand() && onCompleteCalled.compareAndSet(false, true)) {
+            downstream.onComplete();
+        }
+    }
+
+    private class StreamSubscription implements Flow.Subscription {
+        @Override
+        public void request(long n) {
+            if (n > 0L) {
+                demand.updateAndGet(i -> {
+                    var sum = i + n;
+                    return sum >= 0 ? sum : Long.MAX_VALUE;
+                });
+                if (upstream == null) {
+                    // wait for upstream to subscribe before forwarding request
+                    return;
+                }
+                if (upstreamIsRunning()) {
+                    requestOnMlThread(n);
+                } else if (error.get() != null && onErrorCalled.compareAndSet(false, true)) {
+                    downstream.onError(error.get());
+                } else if (onCompleteCalled.compareAndSet(false, true)) {
+                    downstream.onComplete();
+                }
+            } else {
+                cancel();
+                downstream.onError(new IllegalStateException("Cannot request a negative number."));
+            }
+        }
+
+        private boolean upstreamIsRunning() {
+            return isDone.get() == false && error.get() == null;
+        }
+
+        private void requestOnMlThread(long n) {
+            var currentThreadPool = EsExecutors.executorName(Thread.currentThread().getName());
+            if (UTILITY_THREAD_POOL_NAME.equalsIgnoreCase(currentThreadPool)) {
+                upstream.request(n);
+            } else {
+                CompletableFuture.runAsync(() -> upstream.request(n), threadPool.executor(UTILITY_THREAD_POOL_NAME));
+            }
+        }
+
+        @Override
+        public void cancel() {
+            if (upstream != null && upstreamIsRunning()) {
+                upstream.cancel();
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java
index 4ba5b552f802a..0241dcd6142a6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java
@@ -8,23 +8,48 @@
 package org.elasticsearch.xpack.inference.external.googleaistudio;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler;
 import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser;
 import org.elasticsearch.xpack.inference.external.http.retry.RetryException;
 import org.elasticsearch.xpack.inference.external.request.Request;
 import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioErrorResponseEntity;
+import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser;
+import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor;
 import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
 
+import java.io.IOException;
+import java.util.concurrent.Flow;
+
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody;
 
 public class GoogleAiStudioResponseHandler extends BaseResponseHandler {
 
     static final String GOOGLE_AI_STUDIO_UNAVAILABLE = "The Google AI Studio service may be temporarily overloaded or down";
+    private final boolean canHandleStreamingResponses;
+    private final CheckedFunction<XContentParser, String, IOException> content;
 
     public GoogleAiStudioResponseHandler(String requestType, ResponseParser parseFunction) {
+        this(requestType, parseFunction, false, xContentParser -> {
+            assert false : "do not call this";
+            return "";
+        });
+    }
+
+    public GoogleAiStudioResponseHandler(
+        String requestType,
+        ResponseParser parseFunction,
+        boolean canHandleStreamingResponses,
+        CheckedFunction<XContentParser, String, IOException> content
+    ) {
         super(requestType, parseFunction, GoogleAiStudioErrorResponseEntity::fromResponse);
+        this.canHandleStreamingResponses = canHandleStreamingResponses;
+        this.content = content;
     }
 
     @Override
@@ -72,4 +97,18 @@ private static String resourceNotFoundError(Request request) {
         return format("Resource not found at [%s]", request.getURI());
     }
 
+    @Override
+    public boolean canHandleStreamingResponses() {
+        return canHandleStreamingResponses;
+    }
+
+    @Override
+    public InferenceServiceResults parseResult(Request request, Flow.Publisher<HttpResult> flow) {
+        var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser());
+        var googleAiProcessor = new GoogleAiStudioStreamingProcessor(content);
+        flow.subscribe(serverSentEventProcessor);
+        serverSentEventProcessor.subscribe(googleAiProcessor);
+        return new StreamingChatCompletionResults(googleAiProcessor);
+    }
+
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessor.java
new file mode 100644
index 0000000000000..aa1232f4182e3
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessor.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.googleaistudio;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
+import org.elasticsearch.xpack.inference.common.DelegatingProcessor;
+import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent;
+import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+
+class GoogleAiStudioStreamingProcessor extends DelegatingProcessor<Deque<ServerSentEvent>, StreamingChatCompletionResults.Results> {
+    private static final Logger log = LogManager.getLogger(GoogleAiStudioStreamingProcessor.class);
+    private final CheckedFunction<XContentParser, String, IOException> content;
+
+    GoogleAiStudioStreamingProcessor(CheckedFunction<XContentParser, String, IOException> content) {
+        this.content = content;
+    }
+
+    @Override
+    protected void next(Deque<ServerSentEvent> item) throws Exception {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+        var results = new ArrayDeque<StreamingChatCompletionResults.Result>(item.size());
+        for (ServerSentEvent event : item) {
+            if (ServerSentEventField.DATA == event.name() && event.hasValue()) {
+                try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, event.value())) {
+                    var delta = content.apply(jsonParser);
+                    results.offer(new StreamingChatCompletionResults.Result(delta));
+                } catch (Exception e) {
+                    log.warn("Failed to parse event from inference provider: {}", event);
+                    throw e;
+                }
+            }
+        }
+
+        if (results.isEmpty()) {
+            upstream().request(1);
+        } else {
+            downstream().onNext(new StreamingChatCompletionResults.Results(results));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
index 6b04b66cb7c11..f0102d01b37a1 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
@@ -153,44 +153,31 @@ public void stream(HttpRequest request, HttpContext context, ActionListener<Flow
         // The caller must call start() first before attempting to send a request
         assert status.get() == Status.STARTED : "call start() before attempting to send a request";
 
-        // apache can sometimes send us the same error in the consumer and the callback
-        // sometimes it sends us an error just on the callback
-        // notifyOnce will dedupe for us
-        var callOnceListener = ActionListener.notifyOnce(listener);
-
-        SocketAccess.doPrivileged(
-            () -> client.execute(
-                request.requestProducer(),
-                new StreamingHttpResultPublisher(threadPool, settings, callOnceListener),
-                context,
-                new FutureCallback<>() {
-                    @Override
-                    public void completed(HttpResponse response) {
-                        // StreamingHttpResultPublisher will publish results to the Flow.Publisher returned in the ActionListener
-                    }
-
-                    @Override
-                    public void failed(Exception ex) {
-                        throttlerManager.warn(
-                            logger,
-                            format("Request from inference entity id [%s] failed", request.inferenceEntityId()),
-                            ex
-                        );
-                        failUsingUtilityThread(ex, callOnceListener);
-                    }
-
-                    @Override
-                    public void cancelled() {
-                        failUsingUtilityThread(
+        var streamingProcessor = new StreamingHttpResultPublisher(threadPool, settings, listener);
+
+        SocketAccess.doPrivileged(() -> client.execute(request.requestProducer(), streamingProcessor, context, new FutureCallback<>() {
+            @Override
+            public void completed(HttpResponse response) {
+                streamingProcessor.close();
+            }
+
+            @Override
+            public void failed(Exception ex) {
+                threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> streamingProcessor.failed(ex));
+            }
+
+            @Override
+            public void cancelled() {
+                threadPool.executor(UTILITY_THREAD_POOL_NAME)
+                    .execute(
+                        () -> streamingProcessor.failed(
                             new CancellationException(
                                 format("Request from inference entity id [%s] was cancelled", request.inferenceEntityId())
-                            ),
-                            callOnceListener
-                        );
-                    }
-                }
-            )
-        );
+                            )
+                        )
+                    );
+            }
+        }));
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java
index 3fd5d02ef4679..0b2268a448c8a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java
@@ -65,7 +65,7 @@ class StreamingHttpResultPublisher implements HttpAsyncResponseConsumer<HttpResp
 
     StreamingHttpResultPublisher(ThreadPool threadPool, HttpSettings settings, ActionListener<Flow.Publisher<HttpResult>> listener) {
         this.settings = Objects.requireNonNull(settings);
-        this.listener = Objects.requireNonNull(listener);
+        this.listener = ActionListener.notifyOnce(Objects.requireNonNull(listener));
 
         this.taskRunner = new RequestBasedTaskRunner(new OffloadThread(), threadPool, UTILITY_THREAD_POOL_NAME);
     }
@@ -96,11 +96,10 @@ public void consumeContent(ContentDecoder contentDecoder, IOControl ioControl) t
 
         try {
             var consumed = inputBuffer.consumeContent(contentDecoder);
-            var allBytes = new byte[consumed];
-            inputBuffer.read(allBytes);
-
-            // we can have empty bytes, don't bother sending them
-            if (allBytes.length > 0) {
+            // we could have read 0 bytes if the body was delayed getting in, we need to return out so apache can load the body/footer
+            if (consumed > 0) {
+                var allBytes = new byte[consumed];
+                inputBuffer.read(allBytes);
                 queue.offer(() -> {
                     subscriber.onNext(new HttpResult(response, allBytes));
                     var currentBytesInQueue = bytesInQueue.updateAndGet(current -> Long.max(0, current - allBytes.length));
@@ -111,18 +110,17 @@ public void consumeContent(ContentDecoder contentDecoder, IOControl ioControl) t
                         }
                     }
                 });
-            }
 
-            // always check if totalByteSize > the configured setting in case the settings change
-            if (bytesInQueue.accumulateAndGet(allBytes.length, Long::sum) >= settings.getMaxResponseSize().getBytes()) {
-                pauseProducer(ioControl);
-            }
+                // always check if totalByteSize > the configured setting in case the settings change
+                if (bytesInQueue.accumulateAndGet(allBytes.length, Long::sum) >= settings.getMaxResponseSize().getBytes()) {
+                    pauseProducer(ioControl);
+                }
 
-            // always run in case we're waking up from a pause and need to start a new thread
-            taskRunner.requestNextRun();
+                taskRunner.requestNextRun();
 
-            if (listenerCalled.compareAndSet(false, true)) {
-                listener.onResponse(this);
+                if (listenerCalled.compareAndSet(false, true)) {
+                    listener.onResponse(this);
+                }
             }
         } finally {
             inputBuffer.reset();
@@ -152,9 +150,13 @@ public void responseCompleted(HttpContext httpContext) {}
     @Override
     public void failed(Exception e) {
         if (this.isDone.compareAndSet(false, true)) {
-            ex = e;
-            queue.offer(() -> subscriber.onError(e));
-            taskRunner.requestNextRun();
+            if (listenerCalled.compareAndSet(false, true)) {
+                listener.onFailure(e);
+            } else {
+                ex = e;
+                queue.offer(() -> subscriber.onError(e));
+                taskRunner.requestNextRun();
+            }
         }
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java
index 1c6bb58717942..69a5c665feb86 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.xpack.inference.external.response.amazonbedrock.completion.AmazonBedrockChatCompletionResponseHandler;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionModel;
 
-import java.util.List;
 import java.util.function.Supplier;
 
 public class AmazonBedrockChatCompletionRequestManager extends AmazonBedrockRequestManager {
@@ -45,9 +44,11 @@ public void execute(
         Supplier<Boolean> hasRequestCompletedFunction,
         ActionListener<InferenceServiceResults> listener
     ) {
-        List<String> docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs();
+        var docsOnly = DocumentsOnlyInput.of(inferenceInputs);
+        var docsInput = docsOnly.getInputs();
+        var stream = docsOnly.stream();
         var requestEntity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, docsInput);
-        var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, timeout);
+        var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, timeout, stream);
         var responseHandler = new AmazonBedrockChatCompletionResponseHandler();
 
         try {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java
index 426102f7f2376..abe50c6fae3f9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioCompletionResponseEntity;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
 
-import java.util.List;
 import java.util.Objects;
 import java.util.function.Supplier;
 
@@ -32,7 +31,12 @@ public class GoogleAiStudioCompletionRequestManager extends GoogleAiStudioReques
     private final GoogleAiStudioCompletionModel model;
 
     private static ResponseHandler createCompletionHandler() {
-        return new GoogleAiStudioResponseHandler("google ai studio completion", GoogleAiStudioCompletionResponseEntity::fromResponse);
+        return new GoogleAiStudioResponseHandler(
+            "google ai studio completion",
+            GoogleAiStudioCompletionResponseEntity::fromResponse,
+            true,
+            GoogleAiStudioCompletionResponseEntity::content
+        );
     }
 
     public GoogleAiStudioCompletionRequestManager(GoogleAiStudioCompletionModel model, ThreadPool threadPool) {
@@ -47,8 +51,7 @@ public void execute(
         Supplier<Boolean> hasRequestCompletedFunction,
         ActionListener<InferenceServiceResults> listener
     ) {
-        List<String> docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs();
-        GoogleAiStudioCompletionRequest request = new GoogleAiStudioCompletionRequest(docsInput, model);
+        GoogleAiStudioCompletionRequest request = new GoogleAiStudioCompletionRequest(DocumentsOnlyInput.of(inferenceInputs), model);
         execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java
index 803bae40b33ed..6e006fe255956 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java
@@ -110,8 +110,6 @@ public class OpenAiStreamingProcessor extends DelegatingProcessor<Deque<ServerSe
     private static final String CHOICES_FIELD = "choices";
     private static final String DELTA_FIELD = "delta";
     private static final String CONTENT_FIELD = "content";
-    private static final String FINISH_REASON_FIELD = "finish_reason";
-    private static final String STOP_MESSAGE = "stop";
     private static final String DONE_MESSAGE = "[done]";
 
     @Override
@@ -162,21 +160,27 @@ private Iterator<StreamingChatCompletionResults.Result> parse(XContentParserConf
                 ensureExpectedToken(XContentParser.Token.START_OBJECT, currentToken, parser);
 
                 currentToken = parser.nextToken();
-                if (currentToken == XContentParser.Token.END_OBJECT) {
-                    consumeUntilObjectEnd(parser); // end choices
-                    return ""; // stopped
-                }
 
-                if (currentToken == XContentParser.Token.FIELD_NAME && parser.currentName().equals(CONTENT_FIELD)) {
-                    parser.nextToken();
-                } else {
-                    positionParserAtTokenAfterField(parser, CONTENT_FIELD, FAILED_TO_FIND_FIELD_TEMPLATE);
+                // continue until the end of delta
+                while (currentToken != null && currentToken != XContentParser.Token.END_OBJECT) {
+                    if (currentToken == XContentParser.Token.START_OBJECT || currentToken == XContentParser.Token.START_ARRAY) {
+                        parser.skipChildren();
+                    }
+
+                    if (currentToken == XContentParser.Token.FIELD_NAME && parser.currentName().equals(CONTENT_FIELD)) {
+                        parser.nextToken();
+                        ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.currentToken(), parser);
+                        var content = parser.text();
+                        consumeUntilObjectEnd(parser); // end delta
+                        consumeUntilObjectEnd(parser); // end choices
+                        return content;
+                    }
+
+                    currentToken = parser.nextToken();
                 }
-                ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.currentToken(), parser);
-                var content = parser.text();
-                consumeUntilObjectEnd(parser); // end delta
+
                 consumeUntilObjectEnd(parser); // end choices
-                return content;
+                return ""; // stopped
             }).stream()
                 .filter(Objects::nonNull)
                 .filter(Predicate.not(String::isEmpty))
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntity.java
deleted file mode 100644
index aff01316838f8..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntity.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
-
-import org.elasticsearch.core.Nullable;
-
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
-
-public record AmazonBedrockAI21LabsCompletionRequestEntity(
-    List<String> messages,
-    @Nullable Double temperature,
-    @Nullable Double topP,
-    @Nullable Integer maxTokenCount
-) implements AmazonBedrockConverseRequestEntity {
-
-    public AmazonBedrockAI21LabsCompletionRequestEntity {
-        Objects.requireNonNull(messages);
-    }
-
-    @Override
-    public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) {
-        return request.messages(getConverseMessageList(messages));
-    }
-
-    @Override
-    public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) {
-        if (temperature == null && topP == null && maxTokenCount == null) {
-            return request;
-        }
-
-        return request.inferenceConfig(config -> {
-            if (temperature != null) {
-                config.temperature(temperature.floatValue());
-            }
-
-            if (topP != null) {
-                config.topP(topP.floatValue());
-            }
-
-            if (maxTokenCount != null) {
-                config.maxTokens(maxTokenCount);
-            }
-        });
-    }
-
-    @Override
-    public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) {
-        return request;
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntity.java
deleted file mode 100644
index 540012c221192..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntity.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
-
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.Strings;
-
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
-
-public record AmazonBedrockAnthropicCompletionRequestEntity(
-    List<String> messages,
-    @Nullable Double temperature,
-    @Nullable Double topP,
-    @Nullable Double topK,
-    @Nullable Integer maxTokenCount
-) implements AmazonBedrockConverseRequestEntity {
-
-    public AmazonBedrockAnthropicCompletionRequestEntity {
-        Objects.requireNonNull(messages);
-    }
-
-    @Override
-    public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) {
-        return request.messages(getConverseMessageList(messages));
-    }
-
-    @Override
-    public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) {
-        if (temperature == null && topP == null && maxTokenCount == null) {
-            return request;
-        }
-
-        return request.inferenceConfig(config -> {
-            if (temperature != null) {
-                config.temperature(temperature.floatValue());
-            }
-
-            if (topP != null) {
-                config.topP(topP.floatValue());
-            }
-
-            if (maxTokenCount != null) {
-                config.maxTokens(maxTokenCount);
-            }
-        });
-    }
-
-    @Override
-    public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) {
-        if (topK == null) {
-            return request;
-        }
-
-        String topKField = Strings.format("{\"top_k\":%f}", topK.floatValue());
-        return request.additionalModelResponseFieldPaths(topKField);
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java
index f86d2229d42ad..db902290ba0be 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java
@@ -12,6 +12,8 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.additionalTopK;
+
 public final class AmazonBedrockChatCompletionEntityFactory {
     public static AmazonBedrockConverseRequestEntity createEntity(AmazonBedrockChatCompletionModel model, List<String> messages) {
         Objects.requireNonNull(model);
@@ -19,55 +21,21 @@ public static AmazonBedrockConverseRequestEntity createEntity(AmazonBedrockChatC
         var serviceSettings = model.getServiceSettings();
         var taskSettings = model.getTaskSettings();
         switch (serviceSettings.provider()) {
-            case AI21LABS -> {
-                return new AmazonBedrockAI21LabsCompletionRequestEntity(
-                    messages,
-                    taskSettings.temperature(),
-                    taskSettings.topP(),
-                    taskSettings.maxNewTokens()
-                );
-            }
-            case AMAZONTITAN -> {
-                return new AmazonBedrockTitanCompletionRequestEntity(
-                    messages,
-                    taskSettings.temperature(),
-                    taskSettings.topP(),
-                    taskSettings.maxNewTokens()
-                );
-            }
-            case ANTHROPIC -> {
-                return new AmazonBedrockAnthropicCompletionRequestEntity(
+            case AI21LABS, AMAZONTITAN, META -> {
+                return new AmazonBedrockConverseRequestEntity(
                     messages,
                     taskSettings.temperature(),
                     taskSettings.topP(),
-                    taskSettings.topK(),
                     taskSettings.maxNewTokens()
                 );
             }
-            case COHERE -> {
-                return new AmazonBedrockCohereCompletionRequestEntity(
+            case ANTHROPIC, COHERE, MISTRAL -> {
+                return new AmazonBedrockConverseRequestEntity(
                     messages,
                     taskSettings.temperature(),
                     taskSettings.topP(),
-                    taskSettings.topK(),
-                    taskSettings.maxNewTokens()
-                );
-            }
-            case META -> {
-                return new AmazonBedrockMetaCompletionRequestEntity(
-                    messages,
-                    taskSettings.temperature(),
-                    taskSettings.topP(),
-                    taskSettings.maxNewTokens()
-                );
-            }
-            case MISTRAL -> {
-                return new AmazonBedrockMistralCompletionRequestEntity(
-                    messages,
-                    taskSettings.temperature(),
-                    taskSettings.topP(),
-                    taskSettings.topK(),
-                    taskSettings.maxNewTokens()
+                    taskSettings.maxNewTokens(),
+                    additionalTopK(taskSettings.topK())
                 );
             }
             default -> {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java
index 61e0504732462..05d7d90873a71 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java
@@ -8,7 +8,9 @@
 package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
 
 import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest;
 
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.TaskType;
@@ -20,19 +22,26 @@
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.concurrent.Flow;
+
+import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
+import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.inferenceConfig;
 
 public class AmazonBedrockChatCompletionRequest extends AmazonBedrockRequest {
     public static final String USER_ROLE = "user";
     private final AmazonBedrockConverseRequestEntity requestEntity;
     private AmazonBedrockChatCompletionResponseListener listener;
+    private final boolean stream;
 
     public AmazonBedrockChatCompletionRequest(
         AmazonBedrockChatCompletionModel model,
         AmazonBedrockConverseRequestEntity requestEntity,
-        @Nullable TimeValue timeout
+        @Nullable TimeValue timeout,
+        boolean stream
     ) {
         super(model, timeout);
         this.requestEntity = Objects.requireNonNull(requestEntity);
+        this.stream = stream;
     }
 
     @Override
@@ -52,10 +61,16 @@ public TaskType taskType() {
     }
 
     private ConverseRequest getConverseRequest() {
-        var converseRequest = ConverseRequest.builder().modelId(amazonBedrockModel.model());
-        converseRequest = requestEntity.addMessages(converseRequest);
-        converseRequest = requestEntity.addInferenceConfig(converseRequest);
-        converseRequest = requestEntity.addAdditionalModelFields(converseRequest);
+        var converseRequest = ConverseRequest.builder()
+            .modelId(amazonBedrockModel.model())
+            .messages(getConverseMessageList(requestEntity.messages()))
+            .additionalModelResponseFieldPaths(requestEntity.additionalModelFields());
+
+        inferenceConfig(requestEntity).ifPresent(converseRequest::inferenceConfig);
+
+        if (requestEntity.additionalModelFields() != null) {
+            converseRequest.additionalModelResponseFieldPaths(requestEntity.additionalModelFields());
+        }
         return converseRequest.build();
     }
 
@@ -66,4 +81,22 @@ public void executeChatCompletionRequest(
         this.listener = chatCompletionResponseListener;
         this.executeRequest(awsBedrockClient);
     }
+
+    public Flow.Publisher<? extends ChunkedToXContent> executeStreamChatCompletionRequest(AmazonBedrockBaseClient awsBedrockClient) {
+        var converseStreamRequest = ConverseStreamRequest.builder()
+            .modelId(amazonBedrockModel.model())
+            .messages(getConverseMessageList(requestEntity.messages()));
+
+        inferenceConfig(requestEntity).ifPresent(converseStreamRequest::inferenceConfig);
+
+        if (requestEntity.additionalModelFields() != null) {
+            converseStreamRequest.additionalModelResponseFieldPaths(requestEntity.additionalModelFields());
+        }
+        return awsBedrockClient.converseStream(converseStreamRequest.build());
+    }
+
+    @Override
+    public boolean isStreaming() {
+        return stream;
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntity.java
deleted file mode 100644
index f1ae04ad39516..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntity.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
-
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.Strings;
-
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
-
-public record AmazonBedrockCohereCompletionRequestEntity(
-    List<String> messages,
-    @Nullable Double temperature,
-    @Nullable Double topP,
-    @Nullable Double topK,
-    @Nullable Integer maxTokenCount
-) implements AmazonBedrockConverseRequestEntity {
-
-    public AmazonBedrockCohereCompletionRequestEntity {
-        Objects.requireNonNull(messages);
-    }
-
-    @Override
-    public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) {
-        return request.messages(getConverseMessageList(messages));
-    }
-
-    @Override
-    public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) {
-        if (temperature == null && topP == null && maxTokenCount == null) {
-            return request;
-        }
-
-        return request.inferenceConfig(config -> {
-            if (temperature != null) {
-                config.temperature(temperature.floatValue());
-            }
-
-            if (topP != null) {
-                config.topP(topP.floatValue());
-            }
-
-            if (maxTokenCount != null) {
-                config.maxTokens(maxTokenCount);
-            }
-        });
-    }
-
-    @Override
-    public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) {
-        if (topK == null) {
-            return request;
-        }
-
-        String topKField = Strings.format("{\"top_k\":%f}", topK.floatValue());
-        return request.additionalModelResponseFieldPaths(topKField);
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java
index d8e9fa43797cd..203b2820ab16f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java
@@ -7,12 +7,23 @@
 
 package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
 
-import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
+import org.elasticsearch.core.Nullable;
 
-public interface AmazonBedrockConverseRequestEntity {
-    ConverseRequest.Builder addMessages(ConverseRequest.Builder request);
+import java.util.List;
 
-    ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request);
-
-    ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request);
+public record AmazonBedrockConverseRequestEntity(
+    List<String> messages,
+    @Nullable Double temperature,
+    @Nullable Double topP,
+    @Nullable Integer maxTokenCount,
+    @Nullable List<String> additionalModelFields
+) {
+    public AmazonBedrockConverseRequestEntity(
+        List<String> messages,
+        @Nullable Double temperature,
+        @Nullable Double topP,
+        @Nullable Integer maxTokenCount
+    ) {
+        this(messages, temperature, topP, maxTokenCount, null);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java
index 22e0d26a315a7..eb1652ff7ff6d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java
@@ -8,9 +8,14 @@
 package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
 
 import software.amazon.awssdk.services.bedrockruntime.model.ContentBlock;
+import software.amazon.awssdk.services.bedrockruntime.model.InferenceConfiguration;
 import software.amazon.awssdk.services.bedrockruntime.model.Message;
 
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Strings;
+
 import java.util.List;
+import java.util.Optional;
 
 import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockChatCompletionRequest.USER_ROLE;
 
@@ -22,4 +27,32 @@ public static List<Message> getConverseMessageList(List<String> texts) {
             .map(content -> Message.builder().role(USER_ROLE).content(content).build())
             .toList();
     }
+
+    public static Optional<InferenceConfiguration> inferenceConfig(AmazonBedrockConverseRequestEntity request) {
+        if (request.temperature() != null || request.topP() != null || request.maxTokenCount() != null) {
+            var builder = InferenceConfiguration.builder();
+            if (request.temperature() != null) {
+                builder.temperature(request.temperature().floatValue());
+            }
+
+            if (request.topP() != null) {
+                builder.topP(request.topP().floatValue());
+            }
+
+            if (request.maxTokenCount() != null) {
+                builder.maxTokens(request.maxTokenCount());
+            }
+            return Optional.of(builder.build());
+        }
+        return Optional.empty();
+    }
+
+    @Nullable
+    public static List<String> additionalTopK(@Nullable Double topK) {
+        if (topK == null) {
+            return null;
+        }
+
+        return List.of(Strings.format("{\"top_k\":%f}", topK.floatValue()));
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntity.java
deleted file mode 100644
index c21791ced02cb..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntity.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
-
-import org.elasticsearch.core.Nullable;
-
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
-
-public record AmazonBedrockMetaCompletionRequestEntity(
-    List<String> messages,
-    @Nullable Double temperature,
-    @Nullable Double topP,
-    @Nullable Integer maxTokenCount
-) implements AmazonBedrockConverseRequestEntity {
-
-    public AmazonBedrockMetaCompletionRequestEntity {
-        Objects.requireNonNull(messages);
-    }
-
-    @Override
-    public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) {
-        return request.messages(getConverseMessageList(messages));
-    }
-
-    @Override
-    public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) {
-        if (temperature == null && topP == null && maxTokenCount == null) {
-            return request;
-        }
-
-        return request.inferenceConfig(config -> {
-            if (temperature != null) {
-                config.temperature(temperature.floatValue());
-            }
-
-            if (topP != null) {
-                config.topP(topP.floatValue());
-            }
-
-            if (maxTokenCount != null) {
-                config.maxTokens(maxTokenCount);
-            }
-        });
-    }
-
-    @Override
-    public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) {
-        return request;
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntity.java
deleted file mode 100644
index 15931674cbabb..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntity.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
-
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.Strings;
-
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
-
-public record AmazonBedrockMistralCompletionRequestEntity(
-    List<String> messages,
-    @Nullable Double temperature,
-    @Nullable Double topP,
-    @Nullable Double topK,
-    @Nullable Integer maxTokenCount
-) implements AmazonBedrockConverseRequestEntity {
-
-    public AmazonBedrockMistralCompletionRequestEntity {
-        Objects.requireNonNull(messages);
-    }
-
-    @Override
-    public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) {
-        return request.messages(getConverseMessageList(messages));
-    }
-
-    @Override
-    public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) {
-        if (temperature == null && topP == null && maxTokenCount == null) {
-            return request;
-        }
-
-        return request.inferenceConfig(config -> {
-            if (temperature != null) {
-                config.temperature(temperature.floatValue());
-            }
-
-            if (topP != null) {
-                config.topP(topP.floatValue());
-            }
-
-            if (maxTokenCount != null) {
-                config.maxTokens(maxTokenCount);
-            }
-        });
-    }
-
-    @Override
-    public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) {
-        if (topK == null) {
-            return request;
-        }
-
-        String topKField = Strings.format("{\"top_k\":%f}", topK.floatValue());
-        return request.additionalModelResponseFieldPaths(topKField);
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntity.java
deleted file mode 100644
index e267592dfd0ba..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntity.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
-
-import org.elasticsearch.core.Nullable;
-
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
-
-public record AmazonBedrockTitanCompletionRequestEntity(
-    List<String> messages,
-    @Nullable Double temperature,
-    @Nullable Double topP,
-    @Nullable Integer maxTokenCount
-) implements AmazonBedrockConverseRequestEntity {
-
-    public AmazonBedrockTitanCompletionRequestEntity {
-        Objects.requireNonNull(messages);
-    }
-
-    @Override
-    public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) {
-        return request.messages(getConverseMessageList(messages));
-    }
-
-    @Override
-    public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) {
-        if (temperature == null && topP == null && maxTokenCount == null) {
-            return request;
-        }
-
-        return request.inferenceConfig(config -> {
-            if (temperature != null) {
-                config.temperature(temperature.floatValue());
-            }
-
-            if (topP != null) {
-                config.topP(topP.floatValue());
-            }
-
-            if (maxTokenCount != null) {
-                config.maxTokens(maxTokenCount);
-            }
-        });
-    }
-
-    @Override
-    public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) {
-        return request;
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java
index f52fe623e7918..80770d63ef139 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java
@@ -11,46 +11,63 @@
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.ByteArrayEntity;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
 import org.elasticsearch.xpack.inference.external.request.HttpRequest;
 import org.elasticsearch.xpack.inference.external.request.Request;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
 
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
-import java.util.List;
 import java.util.Objects;
 
 public class GoogleAiStudioCompletionRequest implements GoogleAiStudioRequest {
+    private static final String ALT_PARAM = "alt";
+    private static final String SSE_VALUE = "sse";
 
-    private final List<String> input;
+    private final DocumentsOnlyInput input;
 
-    private final URI uri;
+    private final LazyInitializable<URI, RuntimeException> uri;
 
     private final GoogleAiStudioCompletionModel model;
 
-    public GoogleAiStudioCompletionRequest(List<String> input, GoogleAiStudioCompletionModel model) {
-        this.input = input;
+    public GoogleAiStudioCompletionRequest(DocumentsOnlyInput input, GoogleAiStudioCompletionModel model) {
+        this.input = Objects.requireNonNull(input);
         this.model = Objects.requireNonNull(model);
-        this.uri = model.uri();
+        this.uri = new LazyInitializable<>(() -> model.uri(input.stream()));
     }
 
     @Override
     public HttpRequest createHttpRequest() {
-        var httpPost = new HttpPost(uri);
-        var requestEntity = Strings.toString(new GoogleAiStudioCompletionRequestEntity(input));
+        var httpPost = createHttpPost();
+        var requestEntity = Strings.toString(new GoogleAiStudioCompletionRequestEntity(input.getInputs()));
 
         ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8));
         httpPost.setEntity(byteEntity);
         httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType());
-        GoogleAiStudioRequest.decorateWithApiKeyParameter(httpPost, model.getSecretSettings());
 
         return new HttpRequest(httpPost, getInferenceEntityId());
     }
 
+    private HttpPost createHttpPost() {
+        try {
+            var uriBuilder = GoogleAiStudioRequest.builderWithApiKeyParameter(uri.getOrCompute(), model.getSecretSettings());
+            if (isStreaming()) {
+                uriBuilder.addParameter(ALT_PARAM, SSE_VALUE);
+            }
+            return new HttpPost(uriBuilder.build());
+        } catch (Exception e) {
+            ValidationException validationException = new ValidationException(e);
+            validationException.addValidationError(e.getMessage());
+            throw validationException;
+        }
+    }
+
     @Override
     public URI getURI() {
-        return this.uri;
+        return uri.getOrCompute();
     }
 
     @Override
@@ -69,4 +86,9 @@ public boolean[] getTruncationInfo() {
     public String getInferenceEntityId() {
         return model.getInferenceEntityId();
     }
+
+    @Override
+    public boolean isStreaming() {
+        return input.stream();
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java
index fb99deabc9c5e..45403fb8e507d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java
@@ -13,20 +13,15 @@
 import org.elasticsearch.xpack.inference.external.request.Request;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
+import java.net.URI;
+
 public interface GoogleAiStudioRequest extends Request {
 
     String API_KEY_PARAMETER = "key";
 
     static void decorateWithApiKeyParameter(HttpPost httpPost, DefaultSecretSettings secretSettings) {
         try {
-            var uri = httpPost.getURI();
-            var uriWithApiKey = new URIBuilder().setScheme(uri.getScheme())
-                .setHost(uri.getHost())
-                .setPort(uri.getPort())
-                .setPath(uri.getPath())
-                .addParameter(API_KEY_PARAMETER, secretSettings.apiKey().toString())
-                .build();
-
+            var uriWithApiKey = builderWithApiKeyParameter(httpPost.getURI(), secretSettings).build();
             httpPost.setURI(uriWithApiKey);
         } catch (Exception e) {
             ValidationException validationException = new ValidationException(e);
@@ -35,4 +30,12 @@ static void decorateWithApiKeyParameter(HttpPost httpPost, DefaultSecretSettings
         }
     }
 
+    static URIBuilder builderWithApiKeyParameter(URI uri, DefaultSecretSettings secretSettings) {
+        return new URIBuilder().setScheme(uri.getScheme())
+            .setHost(uri.getHost())
+            .setPort(uri.getPort())
+            .setPath(uri.getPath())
+            .addParameter(API_KEY_PARAMETER, secretSettings.apiKey().toString());
+    }
+
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java
index 81ad5b6203682..16c9e7254e108 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java
@@ -17,6 +17,8 @@ public class GoogleAiStudioUtils {
 
     public static final String GENERATE_CONTENT_ACTION = "generateContent";
 
+    public static final String STREAM_GENERATE_CONTENT_ACTION = "streamGenerateContent";
+
     public static final String BATCH_EMBED_CONTENTS_ACTION = "batchEmbedContents";
 
     private GoogleAiStudioUtils() {}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java
index 852f25705d6ff..11dddc78bc469 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java
@@ -77,33 +77,36 @@ public class GoogleAiStudioCompletionResponseEntity {
     public static ChatCompletionResults fromResponse(Request request, HttpResult response) throws IOException {
         var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
         try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) {
-            moveToFirstToken(jsonParser);
+            return new ChatCompletionResults(List.of(new ChatCompletionResults.Result(content(jsonParser))));
+        }
+    }
 
-            XContentParser.Token token = jsonParser.currentToken();
-            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+    public static String content(XContentParser jsonParser) throws IOException {
+        moveToFirstToken(jsonParser);
 
-            positionParserAtTokenAfterField(jsonParser, "candidates", FAILED_TO_FIND_FIELD_TEMPLATE);
+        XContentParser.Token token = jsonParser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
-            jsonParser.nextToken();
-            ensureExpectedToken(XContentParser.Token.START_OBJECT, jsonParser.currentToken(), jsonParser);
+        positionParserAtTokenAfterField(jsonParser, "candidates", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-            positionParserAtTokenAfterField(jsonParser, "content", FAILED_TO_FIND_FIELD_TEMPLATE);
+        jsonParser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, jsonParser.currentToken(), jsonParser);
 
-            token = jsonParser.currentToken();
-            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+        positionParserAtTokenAfterField(jsonParser, "content", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-            positionParserAtTokenAfterField(jsonParser, "parts", FAILED_TO_FIND_FIELD_TEMPLATE);
+        token = jsonParser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
-            jsonParser.nextToken();
-            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+        positionParserAtTokenAfterField(jsonParser, "parts", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-            positionParserAtTokenAfterField(jsonParser, "text", FAILED_TO_FIND_FIELD_TEMPLATE);
+        jsonParser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
-            XContentParser.Token contentToken = jsonParser.currentToken();
-            ensureExpectedToken(XContentParser.Token.VALUE_STRING, contentToken, jsonParser);
-            String content = jsonParser.text();
+        positionParserAtTokenAfterField(jsonParser, "text", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+        XContentParser.Token contentToken = jsonParser.currentToken();
+        ensureExpectedToken(XContentParser.Token.VALUE_STRING, contentToken, jsonParser);
+        return jsonParser.text();
 
-            return new ChatCompletionResults(List.of(new ChatCompletionResults.Result(content)));
-        }
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
index a5702b38ea3f2..4c07516051287 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.BlockLoader;
+import org.elasticsearch.index.mapper.BlockSourceReader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -32,6 +34,7 @@
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperMergeContext;
+import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.NestedObjectMapper;
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
@@ -86,6 +89,7 @@
 public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper {
     public static final NodeFeature SEMANTIC_TEXT_SEARCH_INFERENCE_ID = new NodeFeature("semantic_text.search_inference_id");
     public static final NodeFeature SEMANTIC_TEXT_DEFAULT_ELSER_2 = new NodeFeature("semantic_text.default_elser_2");
+    public static final NodeFeature SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX = new NodeFeature("semantic_text.in_object_field_fix");
 
     public static final String CONTENT_TYPE = "semantic_text";
     public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;
@@ -398,6 +402,25 @@ public Object getOriginalValue(Map<String, Object> sourceAsMap) {
         return XContentMapValues.extractValue(TEXT_FIELD, fieldValueMap);
     }
 
+    @Override
+    protected void doValidate(MappingLookup mappers) {
+        int parentPathIndex = fullPath().lastIndexOf(leafName());
+        if (parentPathIndex > 0) {
+            // Check that the parent object field allows subobjects.
+            // Subtract one from the parent path index to omit the trailing dot delimiter.
+            ObjectMapper parentMapper = mappers.objectMappers().get(fullPath().substring(0, parentPathIndex - 1));
+            if (parentMapper == null) {
+                throw new IllegalStateException(CONTENT_TYPE + " field [" + fullPath() + "] does not have a parent object mapper");
+            }
+
+            if (parentMapper.subobjects() == ObjectMapper.Subobjects.DISABLED) {
+                throw new IllegalArgumentException(
+                    CONTENT_TYPE + " field [" + fullPath() + "] cannot be in an object field with subobjects disabled"
+                );
+            }
+        }
+    }
+
     public static class SemanticTextFieldType extends SimpleMappedFieldType {
         private final String inferenceId;
         private final String searchInferenceId;
@@ -590,6 +613,122 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes
 
             return baseMessageBuilder.toString();
         }
+
+        @Override
+        public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) {
+            SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name().concat(".text")));
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll());
+        }
+    }
+
+    /**
+     * <p>
+     * Insert or replace the path's value in the map with the provided new value. The map will be modified in-place.
+     * If the complete path does not exist in the map, it will be added to the deepest (sub-)map possible.
+     * </p>
+     * <p>
+     * For example, given the map:
+     * </p>
+     * <pre>
+     * {
+     *   "path1": {
+     *     "path2": {
+     *       "key1": "value1"
+     *     }
+     *   }
+     * }
+     * </pre>
+     * <p>
+     * And the caller wanted to insert {@code "path1.path2.path3.key2": "value2"}, the method would emit the modified map:
+     * </p>
+     * <pre>
+     * {
+     *   "path1": {
+     *     "path2": {
+     *       "key1": "value1",
+     *       "path3.key2": "value2"
+     *     }
+     *   }
+     * }
+     * </pre>
+     *
+     * @param path the value's path in the map.
+     * @param map the map to search and modify in-place.
+     * @param newValue the new value to assign to the path.
+     *
+     * @throws IllegalArgumentException If either the path cannot be fully traversed or there is ambiguity about where to insert the new
+     *                                  value.
+     */
+    public static void insertValue(String path, Map<?, ?> map, Object newValue) {
+        String[] pathElements = path.split("\\.");
+        if (pathElements.length == 0) {
+            return;
+        }
+
+        List<SuffixMap> suffixMaps = extractSuffixMaps(pathElements, 0, map);
+        if (suffixMaps.isEmpty()) {
+            // This should never happen. Throw in case it does for some reason.
+            throw new IllegalStateException("extractSuffixMaps returned an empty suffix map list");
+        } else if (suffixMaps.size() == 1) {
+            SuffixMap suffixMap = suffixMaps.getFirst();
+            suffixMap.map().put(suffixMap.suffix(), newValue);
+        } else {
+            throw new IllegalArgumentException(
+                "Path [" + path + "] could be inserted in " + suffixMaps.size() + " distinct ways, it is ambiguous which one to use"
+            );
+        }
+    }
+
+    private record SuffixMap(String suffix, Map<String, Object> map) {}
+
+    private static List<SuffixMap> extractSuffixMaps(String[] pathElements, int index, Object currentValue) {
+        if (currentValue instanceof List<?> valueList) {
+            List<SuffixMap> suffixMaps = new ArrayList<>(valueList.size());
+            for (Object o : valueList) {
+                suffixMaps.addAll(extractSuffixMaps(pathElements, index, o));
+            }
+
+            return suffixMaps;
+        } else if (currentValue instanceof Map<?, ?>) {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> map = (Map<String, Object>) currentValue;
+            List<SuffixMap> suffixMaps = new ArrayList<>(map.size());
+
+            String key = pathElements[index];
+            while (index < pathElements.length) {
+                if (map.containsKey(key)) {
+                    if (index + 1 == pathElements.length) {
+                        // We found the complete path
+                        suffixMaps.add(new SuffixMap(key, map));
+                    } else {
+                        // We've matched that path partially, keep traversing to try to match it fully
+                        suffixMaps.addAll(extractSuffixMaps(pathElements, index + 1, map.get(key)));
+                    }
+                }
+
+                if (++index < pathElements.length) {
+                    key += "." + pathElements[index];
+                }
+            }
+
+            if (suffixMaps.isEmpty()) {
+                // We checked for all remaining elements in the path, and they do not exist. This means we found a leaf map that we should
+                // add the value to.
+                suffixMaps.add(new SuffixMap(key, map));
+            }
+
+            return suffixMaps;
+        } else {
+            throw new IllegalArgumentException(
+                "Path ["
+                    + String.join(".", Arrays.copyOfRange(pathElements, 0, index))
+                    + "] has value ["
+                    + currentValue
+                    + "] of type ["
+                    + currentValue.getClass().getSimpleName()
+                    + "], which cannot be traversed into further"
+            );
+        }
     }
 
     private static ObjectMapper createInferenceField(
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java
index a33a49cc52d94..478f2e6a21868 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java
@@ -132,7 +132,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.SEMANTIC_QUERY;
+        return TransportVersions.V_8_15_0;
     }
 
     public static SemanticQueryBuilder fromXContent(XContentParser parser) throws IOException {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java
index fd0ad220faa3b..e4093a91c2359 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java
@@ -98,7 +98,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.TEXT_SIMILARITY_RERANKER_RETRIEVER;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
index 8bccf6e7d1022..91b6cdc61afe4 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
@@ -10,7 +10,6 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
@@ -20,7 +19,6 @@
 import org.elasticsearch.search.retriever.CompoundRetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverParserContext;
-import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -83,7 +81,7 @@ public static TextSimilarityRankRetrieverBuilder fromXContent(XContentParser par
             throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + TextSimilarityRankBuilder.NAME + "]");
         }
         if (context.clusterSupportsFeature(TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED) == false) {
-            throw new UnsupportedOperationException(
+            throw new IllegalArgumentException(
                 "[text_similarity_reranker] retriever composition feature is not supported by all nodes in the cluster"
             );
         }
@@ -158,33 +156,18 @@ protected RankDoc[] combineInnerRetrieverResults(List<ScoreDoc[]> rankResults) {
         return textSimilarityRankDocs;
     }
 
-    @Override
-    public QueryBuilder explainQuery() {
-        // the original matching set of the TextSimilarityRank retriever is specified by its nested retriever
-        return new RankDocsQueryBuilder(rankDocs, new QueryBuilder[] { innerRetrievers.getFirst().retriever().explainQuery() }, true);
-    }
-
     @Override
     protected SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) {
         var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit)
             .trackTotalHits(false)
             .storedFields(new StoredFieldsContext(false))
             .size(rankWindowSize);
+        // apply the pre-filters downstream once
         if (preFilterQueryBuilders.isEmpty() == false) {
             retrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders);
         }
         retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, true);
 
-        // apply the pre-filters
-        if (preFilterQueryBuilders.size() > 0) {
-            QueryBuilder query = sourceBuilder.query();
-            BoolQueryBuilder newQuery = new BoolQueryBuilder();
-            if (query != null) {
-                newQuery.must(query);
-            }
-            preFilterQueryBuilders.forEach(newQuery::filter);
-            sourceBuilder.query(newQuery);
-        }
         sourceBuilder.rankBuilder(
             new TextSimilarityRankBuilder(this.field, this.inferenceId, this.inferenceText, this.rankWindowSize, this.minScore)
         );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java
index d756c0ef26f14..4506a05d58054 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java
@@ -3,6 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
+ *
+ * this file contains code contributed by a generative AI
  */
 
 package org.elasticsearch.xpack.inference.registry;
@@ -21,14 +23,19 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.elasticsearch.inference.InferenceService;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
@@ -49,15 +56,30 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.core.Strings.format;
 
+/**
+ * Class for persisting and reading inference endpoint configurations.
+ * Some inference services provide default configurations, the registry is
+ * made aware of these at start up via {@link #addDefaultIds(InferenceService.DefaultConfigId)}.
+ * Only the ids and service details are registered at this point
+ * as the full config definition may not be known at start up.
+ * The full config is lazily populated on read and persisted to the
+ * index. This has the effect of creating the backing index on reading
+ * the configs. {@link #getAllModels(boolean, ActionListener)} has an option
+ * to not write the default configs to index on read to avoid index creation.
+ */
 public class ModelRegistry {
     public record ModelConfigMap(Map<String, Object> config, Map<String, Object> secrets) {}
 
@@ -81,27 +103,33 @@ public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap)
     private static final Logger logger = LogManager.getLogger(ModelRegistry.class);
 
     private final OriginSettingClient client;
-    private Map<String, UnparsedModel> defaultConfigs;
+    private final List<InferenceService.DefaultConfigId> defaultConfigIds;
+
+    private final Set<String> preventDeletionLock = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
     public ModelRegistry(Client client) {
         this.client = new OriginSettingClient(client, ClientHelper.INFERENCE_ORIGIN);
-        this.defaultConfigs = new HashMap<>();
+        defaultConfigIds = new ArrayList<>();
     }
 
-    public void addDefaultConfiguration(UnparsedModel serviceDefaultConfig) {
-        if (defaultConfigs.containsKey(serviceDefaultConfig.inferenceEntityId())) {
+    /**
+     * Set the default inference ids provided by the services
+     * @param defaultConfigIds The defaults
+     */
+    public void addDefaultIds(InferenceService.DefaultConfigId defaultConfigIds) {
+        var matched = idMatchedDefault(defaultConfigIds.inferenceId(), this.defaultConfigIds);
+        if (matched.isPresent()) {
             throw new IllegalStateException(
                 "Cannot add default endpoint to the inference endpoint registry with duplicate inference id ["
-                    + serviceDefaultConfig.inferenceEntityId()
+                    + defaultConfigIds.inferenceId()
                     + "] declared by service ["
-                    + serviceDefaultConfig.service()
+                    + defaultConfigIds.service().name()
                     + "]. The inference Id is already use by ["
-                    + defaultConfigs.get(serviceDefaultConfig.inferenceEntityId()).service()
+                    + matched.get().service().name()
                     + "] service."
             );
         }
-
-        defaultConfigs.put(serviceDefaultConfig.inferenceEntityId(), serviceDefaultConfig);
+        this.defaultConfigIds.add(defaultConfigIds);
     }
 
     /**
@@ -110,15 +138,15 @@ public void addDefaultConfiguration(UnparsedModel serviceDefaultConfig) {
      * @param listener Model listener
      */
     public void getModelWithSecrets(String inferenceEntityId, ActionListener<UnparsedModel> listener) {
-        if (defaultConfigs.containsKey(inferenceEntityId)) {
-            listener.onResponse(deepCopyDefaultConfig(defaultConfigs.get(inferenceEntityId)));
-            return;
-        }
-
         ActionListener<SearchResponse> searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> {
-            // There should be a hit for the configurations and secrets
+            // There should be a hit for the configurations
             if (searchResponse.getHits().getHits().length == 0) {
-                delegate.onFailure(inferenceNotFoundException(inferenceEntityId));
+                var maybeDefault = idMatchedDefault(inferenceEntityId, defaultConfigIds);
+                if (maybeDefault.isPresent()) {
+                    getDefaultConfig(true, maybeDefault.get(), listener);
+                } else {
+                    delegate.onFailure(inferenceNotFoundException(inferenceEntityId));
+                }
                 return;
             }
 
@@ -141,15 +169,15 @@ public void getModelWithSecrets(String inferenceEntityId, ActionListener<Unparse
      * @param listener Model listener
      */
     public void getModel(String inferenceEntityId, ActionListener<UnparsedModel> listener) {
-        if (defaultConfigs.containsKey(inferenceEntityId)) {
-            listener.onResponse(deepCopyDefaultConfig(defaultConfigs.get(inferenceEntityId)));
-            return;
-        }
-
         ActionListener<SearchResponse> searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> {
-            // There should be a hit for the configurations and secrets
+            // There should be a hit for the configurations
             if (searchResponse.getHits().getHits().length == 0) {
-                delegate.onFailure(inferenceNotFoundException(inferenceEntityId));
+                var maybeDefault = idMatchedDefault(inferenceEntityId, defaultConfigIds);
+                if (maybeDefault.isPresent()) {
+                    getDefaultConfig(true, maybeDefault.get(), listener);
+                } else {
+                    delegate.onFailure(inferenceNotFoundException(inferenceEntityId));
+                }
                 return;
             }
 
@@ -180,29 +208,9 @@ private ResourceNotFoundException inferenceNotFoundException(String inferenceEnt
      */
     public void getModelsByTaskType(TaskType taskType, ActionListener<List<UnparsedModel>> listener) {
         ActionListener<SearchResponse> searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> {
-            var defaultConfigsForTaskType = defaultConfigs.values()
-                .stream()
-                .filter(m -> m.taskType() == taskType)
-                .map(ModelRegistry::deepCopyDefaultConfig)
-                .toList();
-
-            // Not an error if no models of this task_type
-            if (searchResponse.getHits().getHits().length == 0 && defaultConfigsForTaskType.isEmpty()) {
-                delegate.onResponse(List.of());
-                return;
-            }
-
             var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistry::unparsedModelFromMap).toList();
-
-            if (defaultConfigsForTaskType.isEmpty() == false) {
-                var allConfigs = new ArrayList<UnparsedModel>();
-                allConfigs.addAll(modelConfigs);
-                allConfigs.addAll(defaultConfigsForTaskType);
-                allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId));
-                delegate.onResponse(allConfigs);
-            } else {
-                delegate.onResponse(modelConfigs);
-            }
+            var defaultConfigsForTaskType = taskTypeMatchedDefaults(taskType, defaultConfigIds);
+            addAllDefaultConfigsIfMissing(true, modelConfigs, defaultConfigsForTaskType, delegate);
         });
 
         QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(TASK_TYPE_FIELD, taskType.toString()));
@@ -219,24 +227,20 @@ public void getModelsByTaskType(TaskType taskType, ActionListener<List<UnparsedM
 
     /**
      * Get all models.
+     * If the defaults endpoint configurations have not been persisted then only
+     * persist them if {@code persistDefaultEndpoints == true}. Persisting the
+     * configs has the side effect of creating the index.
+     *
      * Secret settings are not included
+     * @param persistDefaultEndpoints Persist the defaults endpoint configurations if
+     *                                not already persisted. When false this avoids the creation
+     *                                of the backing index.
      * @param listener Models listener
      */
-    public void getAllModels(ActionListener<List<UnparsedModel>> listener) {
+    public void getAllModels(boolean persistDefaultEndpoints, ActionListener<List<UnparsedModel>> listener) {
         ActionListener<SearchResponse> searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> {
-            var defaults = defaultConfigs.values().stream().map(ModelRegistry::deepCopyDefaultConfig).toList();
-
-            if (searchResponse.getHits().getHits().length == 0 && defaults.isEmpty()) {
-                delegate.onResponse(List.of());
-                return;
-            }
-
             var foundConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistry::unparsedModelFromMap).toList();
-            var allConfigs = new ArrayList<UnparsedModel>();
-            allConfigs.addAll(foundConfigs);
-            allConfigs.addAll(defaults);
-            allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId));
-            delegate.onResponse(allConfigs);
+            addAllDefaultConfigsIfMissing(persistDefaultEndpoints, foundConfigs, defaultConfigIds, delegate);
         });
 
         // In theory the index should only contain model config documents
@@ -254,6 +258,76 @@ public void getAllModels(ActionListener<List<UnparsedModel>> listener) {
         client.search(modelSearch, searchListener);
     }
 
+    private void addAllDefaultConfigsIfMissing(
+        boolean persistDefaultEndpoints,
+        List<UnparsedModel> foundConfigs,
+        List<InferenceService.DefaultConfigId> matchedDefaults,
+        ActionListener<List<UnparsedModel>> listener
+    ) {
+        var foundIds = foundConfigs.stream().map(UnparsedModel::inferenceEntityId).collect(Collectors.toSet());
+        var missing = matchedDefaults.stream().filter(d -> foundIds.contains(d.inferenceId()) == false).toList();
+
+        if (missing.isEmpty()) {
+            listener.onResponse(foundConfigs);
+        } else {
+            var groupedListener = new GroupedActionListener<UnparsedModel>(
+                missing.size(),
+                listener.delegateFailure((delegate, listOfModels) -> {
+                    var allConfigs = new ArrayList<UnparsedModel>();
+                    allConfigs.addAll(foundConfigs);
+                    allConfigs.addAll(listOfModels);
+                    allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId));
+                    delegate.onResponse(allConfigs);
+                })
+            );
+
+            for (var required : missing) {
+                getDefaultConfig(persistDefaultEndpoints, required, groupedListener);
+            }
+        }
+    }
+
+    private void getDefaultConfig(
+        boolean persistDefaultEndpoints,
+        InferenceService.DefaultConfigId defaultConfig,
+        ActionListener<UnparsedModel> listener
+    ) {
+        defaultConfig.service().defaultConfigs(listener.delegateFailureAndWrap((delegate, models) -> {
+            boolean foundModel = false;
+            for (var m : models) {
+                if (m.getInferenceEntityId().equals(defaultConfig.inferenceId())) {
+                    foundModel = true;
+                    if (persistDefaultEndpoints) {
+                        storeDefaultEndpoint(m, () -> listener.onResponse(modelToUnparsedModel(m)));
+                    } else {
+                        listener.onResponse(modelToUnparsedModel(m));
+                    }
+                    break;
+                }
+            }
+
+            if (foundModel == false) {
+                listener.onFailure(
+                    new IllegalStateException("Configuration not found for default inference id [" + defaultConfig.inferenceId() + "]")
+                );
+            }
+        }));
+    }
+
+    private void storeDefaultEndpoint(Model preconfigured, Runnable runAfter) {
+        var responseListener = ActionListener.<Boolean>wrap(success -> {
+            logger.debug("Added default inference endpoint [{}]", preconfigured.getInferenceEntityId());
+        }, exception -> {
+            if (exception instanceof ResourceAlreadyExistsException) {
+                logger.debug("Default inference id [{}] already exists", preconfigured.getInferenceEntityId());
+            } else {
+                logger.error("Failed to store default inference id [" + preconfigured.getInferenceEntityId() + "]", exception);
+            }
+        });
+
+        storeModel(preconfigured, ActionListener.runAfter(responseListener, runAfter));
+    }
+
     private ArrayList<ModelConfigMap> parseHitsAsModels(SearchHits hits) {
         var modelConfigs = new ArrayList<ModelConfigMap>();
         for (var hit : hits) {
@@ -306,7 +380,140 @@ private ModelConfigMap createModelConfigMap(SearchHits hits, String inferenceEnt
         );
     }
 
+    public void updateModelTransaction(Model newModel, Model existingModel, ActionListener<Boolean> finalListener) {
+
+        String inferenceEntityId = newModel.getConfigurations().getInferenceEntityId();
+        logger.info("Attempting to store update to inference endpoint [{}]", inferenceEntityId);
+
+        if (preventDeletionLock.contains(inferenceEntityId)) {
+            logger.warn(format("Attempted to update endpoint [{}] that is already being updated", inferenceEntityId));
+            finalListener.onFailure(
+                new ElasticsearchStatusException(
+                    "Endpoint [{}] is currently being updated. Try again once the update completes",
+                    RestStatus.CONFLICT,
+                    inferenceEntityId
+                )
+            );
+            return;
+        } else {
+            preventDeletionLock.add(inferenceEntityId);
+        }
+
+        SubscribableListener.<BulkResponse>newForked((subListener) -> {
+            // in this block, we try to update the stored model configurations
+            IndexRequest configRequest = createIndexRequest(
+                Model.documentId(inferenceEntityId),
+                InferenceIndex.INDEX_NAME,
+                newModel.getConfigurations(),
+                true
+            );
+
+            ActionListener<BulkResponse> storeConfigListener = subListener.delegateResponse((l, e) -> {
+                // this block will only be called if the bulk unexpectedly throws an exception
+                preventDeletionLock.remove(inferenceEntityId);
+                l.onFailure(e);
+            });
+
+            client.prepareBulk().add(configRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).execute(storeConfigListener);
+
+        }).<BulkResponse>andThen((subListener, configResponse) -> {
+            // in this block, we respond to the success or failure of updating the model configurations, then try to store the new secrets
+            if (configResponse.hasFailures()) {
+                // if storing the model configurations failed, it won't throw an exception, we need to check the BulkResponse and handle the
+                // exceptions ourselves.
+                logger.error(
+                    format("Failed to update inference endpoint [%s] due to [%s]", inferenceEntityId, configResponse.buildFailureMessage())
+                );
+                preventDeletionLock.remove(inferenceEntityId);
+                // Since none of our updates succeeded at this point, we can simply return.
+                finalListener.onFailure(
+                    new ElasticsearchStatusException(
+                        format("Failed to update inference endpoint [%s] due to [%s]", inferenceEntityId),
+                        RestStatus.INTERNAL_SERVER_ERROR,
+                        configResponse.buildFailureMessage()
+                    )
+                );
+            } else {
+                // Since the model configurations were successfully updated, we can now try to store the new secrets
+                IndexRequest secretsRequest = createIndexRequest(
+                    Model.documentId(newModel.getConfigurations().getInferenceEntityId()),
+                    InferenceSecretsIndex.INDEX_NAME,
+                    newModel.getSecrets(),
+                    true
+                );
+
+                ActionListener<BulkResponse> storeSecretsListener = subListener.delegateResponse((l, e) -> {
+                    // this block will only be called if the bulk unexpectedly throws an exception
+                    preventDeletionLock.remove(inferenceEntityId);
+                    l.onFailure(e);
+                });
+
+                client.prepareBulk()
+                    .add(secretsRequest)
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                    .execute(storeSecretsListener);
+            }
+        }).<BulkResponse>andThen((subListener, secretsResponse) -> {
+            // in this block, we respond to the success or failure of updating the model secrets
+            if (secretsResponse.hasFailures()) {
+                // since storing the secrets failed, we will try to restore / roll-back-to the previous model configurations
+                IndexRequest configRequest = createIndexRequest(
+                    Model.documentId(inferenceEntityId),
+                    InferenceIndex.INDEX_NAME,
+                    existingModel.getConfigurations(),
+                    true
+                );
+                logger.error(
+                    "Failed to update inference endpoint secrets [{}], attempting rolling back to previous state",
+                    inferenceEntityId
+                );
+
+                ActionListener<BulkResponse> rollbackConfigListener = subListener.delegateResponse((l, e) -> {
+                    // this block will only be called if the bulk unexpectedly throws an exception
+                    preventDeletionLock.remove(inferenceEntityId);
+                    l.onFailure(e);
+                });
+                client.prepareBulk()
+                    .add(configRequest)
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                    .execute(rollbackConfigListener);
+            } else {
+                // since updating the secrets was successful, we can remove the lock and respond to the final listener
+                preventDeletionLock.remove(inferenceEntityId);
+                finalListener.onResponse(true);
+            }
+        }).<BulkResponse>andThen((subListener, configResponse) -> {
+            // this block will be called if the secrets response failed, and the rollback didn't throw an exception.
+            // The rollback still could have failed though, so we need to check for that.
+            preventDeletionLock.remove(inferenceEntityId);
+            if (configResponse.hasFailures()) {
+                logger.error(
+                    format("Failed to update inference endpoint [%s] due to [%s]", inferenceEntityId, configResponse.buildFailureMessage())
+                );
+                finalListener.onFailure(
+                    new ElasticsearchStatusException(
+                        format(
+                            "Failed to rollback while handling failure to update inference endpoint [%s]. "
+                                + "Endpoint may be in an inconsistent state due to [%s]",
+                            inferenceEntityId
+                        ),
+                        RestStatus.INTERNAL_SERVER_ERROR,
+                        configResponse.buildFailureMessage()
+                    )
+                );
+            } else {
+                logger.warn("Failed to update inference endpoint [{}], successfully rolled back to previous state", inferenceEntityId);
+                finalListener.onResponse(false);
+            }
+        });
+
+    }
+
+    /**
+     * Note: storeModel does not overwrite existing models and thus does not need to check the lock
+     */
     public void storeModel(Model model, ActionListener<Boolean> listener) {
+
         ActionListener<BulkResponse> bulkResponseActionListener = getStoreModelListener(model, listener);
 
         IndexRequest configRequest = createIndexRequest(
@@ -405,6 +612,16 @@ private static BulkItemResponse.Failure getFirstBulkFailure(BulkResponse bulkRes
     }
 
     public void deleteModel(String inferenceEntityId, ActionListener<Boolean> listener) {
+        if (preventDeletionLock.contains(inferenceEntityId)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Model is currently being updated, you may delete the model once the update completes",
+                    RestStatus.CONFLICT
+                )
+            );
+            return;
+        }
+
         DeleteByQueryRequest request = new DeleteByQueryRequest().setAbortOnVersionConflict(false);
         request.indices(InferenceIndex.INDEX_PATTERN, InferenceSecretsIndex.INDEX_PATTERN);
         request.setQuery(documentIdQuery(inferenceEntityId));
@@ -428,60 +645,36 @@ private static IndexRequest createIndexRequest(String docId, String indexName, T
         }
     }
 
-    private QueryBuilder documentIdQuery(String inferenceEntityId) {
-        return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(Model.documentId(inferenceEntityId)));
-    }
-
-    static UnparsedModel deepCopyDefaultConfig(UnparsedModel other) {
-        // Because the default config uses immutable maps
-        return new UnparsedModel(
-            other.inferenceEntityId(),
-            other.taskType(),
-            other.service(),
-            copySettingsMap(other.settings()),
-            copySecretsMap(other.secrets())
-        );
-    }
-
-    @SuppressWarnings("unchecked")
-    static Map<String, Object> copySettingsMap(Map<String, Object> other) {
-        var result = new HashMap<String, Object>();
-
-        var serviceSettings = (Map<String, Object>) other.get(ModelConfigurations.SERVICE_SETTINGS);
-        if (serviceSettings != null) {
-            var copiedServiceSettings = copyMap1LevelDeep(serviceSettings);
-            result.put(ModelConfigurations.SERVICE_SETTINGS, copiedServiceSettings);
-        }
+    private static UnparsedModel modelToUnparsedModel(Model model) {
+        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+            model.getConfigurations()
+                .toXContent(builder, new ToXContent.MapParams(Map.of(ModelConfigurations.USE_ID_FOR_INDEX, Boolean.TRUE.toString())));
 
-        var taskSettings = (Map<String, Object>) other.get(ModelConfigurations.TASK_SETTINGS);
-        if (taskSettings != null) {
-            var copiedTaskSettings = copyMap1LevelDeep(taskSettings);
-            result.put(ModelConfigurations.TASK_SETTINGS, copiedTaskSettings);
-        }
+            var modelConfigMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
+            return unparsedModelFromMap(new ModelConfigMap(modelConfigMap, new HashMap<>()));
 
-        var chunkSettings = (Map<String, Object>) other.get(ModelConfigurations.CHUNKING_SETTINGS);
-        if (chunkSettings != null) {
-            var copiedChunkSettings = copyMap1LevelDeep(chunkSettings);
-            result.put(ModelConfigurations.CHUNKING_SETTINGS, copiedChunkSettings);
+        } catch (IOException ex) {
+            throw new ElasticsearchException("[{}] Error serializing inference endpoint configuration", model.getInferenceEntityId(), ex);
         }
+    }
 
-        return result;
+    private QueryBuilder documentIdQuery(String inferenceEntityId) {
+        return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(Model.documentId(inferenceEntityId)));
     }
 
-    static Map<String, Object> copySecretsMap(Map<String, Object> other) {
-        return copyMap1LevelDeep(other);
+    static Optional<InferenceService.DefaultConfigId> idMatchedDefault(
+        String inferenceId,
+        List<InferenceService.DefaultConfigId> defaultConfigIds
+    ) {
+        return defaultConfigIds.stream().filter(defaultConfigId -> defaultConfigId.inferenceId().equals(inferenceId)).findFirst();
     }
 
-    @SuppressWarnings("unchecked")
-    static Map<String, Object> copyMap1LevelDeep(Map<String, Object> other) {
-        var result = new HashMap<String, Object>();
-        for (var entry : other.entrySet()) {
-            if (entry.getValue() instanceof Map<?, ?>) {
-                result.put(entry.getKey(), new HashMap<>((Map<String, Object>) entry.getValue()));
-            } else {
-                result.put(entry.getKey(), entry.getValue());
-            }
-        }
-        return result;
+    static List<InferenceService.DefaultConfigId> taskTypeMatchedDefaults(
+        TaskType taskType,
+        List<InferenceService.DefaultConfigId> defaultConfigIds
+    ) {
+        return defaultConfigIds.stream()
+            .filter(defaultConfigId -> defaultConfigId.taskType().equals(taskType))
+            .collect(Collectors.toList());
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java
index 9f64b58e48b55..2dec72e6692a6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java
@@ -14,6 +14,12 @@ public final class Paths {
     static final String INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}";
     static final String TASK_TYPE_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/{" + INFERENCE_ID + "}";
     static final String INFERENCE_DIAGNOSTICS_PATH = "_inference/.diagnostics";
+    static final String TASK_TYPE_INFERENCE_ID_UPDATE_PATH = "_inference/{"
+        + TASK_TYPE_OR_INFERENCE_ID
+        + "}/{"
+        + INFERENCE_ID
+        + "}/_update";
+    static final String INFERENCE_ID_UPDATE_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/_update";
 
     static final String STREAM_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/_stream";
     static final String STREAM_TASK_TYPE_INFERENCE_ID_PATH = "_inference/{"
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java
new file mode 100644
index 0000000000000..9405a6752538c
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.rest;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction;
+
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+import static org.elasticsearch.xpack.inference.rest.Paths.INFERENCE_ID;
+import static org.elasticsearch.xpack.inference.rest.Paths.INFERENCE_ID_UPDATE_PATH;
+import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_INFERENCE_ID_UPDATE_PATH;
+import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_OR_INFERENCE_ID;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestUpdateInferenceModelAction extends BaseRestHandler {
+    @Override
+    public String getName() {
+        return "update_inference_model_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(PUT, INFERENCE_ID_UPDATE_PATH), new Route(PUT, TASK_TYPE_INFERENCE_ID_UPDATE_PATH));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        String inferenceEntityId;
+        TaskType taskType;
+        if (restRequest.hasParam(INFERENCE_ID)) {
+            inferenceEntityId = restRequest.param(INFERENCE_ID);
+            taskType = TaskType.fromStringOrStatusException(restRequest.param(TASK_TYPE_OR_INFERENCE_ID));
+        } else {
+            throw new ElasticsearchStatusException("Inference ID must be provided in the path", RestStatus.BAD_REQUEST);
+        }
+
+        var request = new UpdateInferenceModelAction.Request(
+            inferenceEntityId,
+            restRequest.requiredContent(),
+            restRequest.getXContentType(),
+            taskType,
+            RestUtils.getMasterNodeTimeout(restRequest)
+        );
+        return channel -> client.execute(UpdateInferenceModelAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java
index 7ae7c5f4c3909..a397da05b1ce4 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasables;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Streams;
 import org.elasticsearch.rest.ChunkedRestResponseBodyPart;
 import org.elasticsearch.rest.RestChannel;
@@ -299,9 +298,7 @@ private ServerSentEventResponseBodyPart(ServerSentEvents event, ChunkedToXConten
             this.xContentBuilder = new LazyInitializable<>(
                 () -> channel.newBuilder(channel.request().getXContentType(), null, true, Streams.noCloseStream(out))
             );
-            this.serialization = channel.request().getRestApiVersion() == RestApiVersion.V_7
-                ? item.toXContentChunkedV7(params)
-                : item.toXContentChunked(params);
+            this.serialization = item.toXContentChunked(channel.request().getRestApiVersion(), params);
         }
 
         @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
index 32c1d17373e53..9e7f8712b4087 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
@@ -435,6 +435,32 @@ public static Integer extractRequiredPositiveIntegerLessThanOrEqualToMax(
         return field;
     }
 
+    public static Integer extractRequiredPositiveIntegerBetween(
+        Map<String, Object> map,
+        String settingName,
+        int minValue,
+        int maxValue,
+        String scope,
+        ValidationException validationException
+    ) {
+        Integer field = extractRequiredPositiveInteger(map, settingName, scope, validationException);
+
+        if (field != null && field < minValue) {
+            validationException.addValidationError(
+                ServiceUtils.mustBeGreaterThanOrEqualNumberErrorMessage(settingName, scope, field, minValue)
+            );
+            return null;
+        }
+        if (field != null && field > maxValue) {
+            validationException.addValidationError(
+                ServiceUtils.mustBeLessThanOrEqualNumberErrorMessage(settingName, scope, field, maxValue)
+            );
+            return null;
+        }
+
+        return field;
+    }
+
     public static Integer extractOptionalPositiveInteger(
         Map<String, Object> map,
         String settingName,
@@ -625,6 +651,40 @@ public static String mustBeAPositiveLongErrorMessage(String settingName, String
         return format("[%s] Invalid value [%s]. [%s] must be a positive long", scope, value, settingName);
     }
 
+    /**
+     * task_type can be specified as either a URL parameter or in the
+     * request body. Resolve which to use or throw if the settings are
+     * inconsistent
+     * @param urlTaskType Taken from the URL parameter. ANY means not specified.
+     * @param bodyTaskType Taken from the request body. Maybe null
+     * @return The resolved task type
+     */
+    public static TaskType resolveTaskType(TaskType urlTaskType, String bodyTaskType) {
+        if (bodyTaskType == null) {
+            if (urlTaskType == TaskType.ANY) {
+                throw new ElasticsearchStatusException("model is missing required setting [task_type]", RestStatus.BAD_REQUEST);
+            } else {
+                return urlTaskType;
+            }
+        }
+
+        TaskType parsedBodyTask = TaskType.fromStringOrStatusException(bodyTaskType);
+        if (parsedBodyTask == TaskType.ANY) {
+            throw new ElasticsearchStatusException("task_type [any] is not valid type for inference", RestStatus.BAD_REQUEST);
+        }
+
+        if (parsedBodyTask.isAnyOrSame(urlTaskType) == false) {
+            throw new ElasticsearchStatusException(
+                "Cannot resolve conflicting task_type parameter in the request URL [{}] and the request body [{}]",
+                RestStatus.BAD_REQUEST,
+                urlTaskType.toString(),
+                bodyTaskType
+            );
+        }
+
+        return parsedBodyTask;
+    }
+
     /**
      * Functional interface for creating an enum from a string.
      * @param <E>
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java
index 63f82a8eceb98..05b5873a81d8d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java
@@ -139,4 +139,12 @@ public int hashCode() {
     public Map<String, Object> getParameters() {
         return parameters;
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AlibabaCloudSearchCompletionTaskSettings updatedSettings = AlibabaCloudSearchCompletionTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, updatedSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java
index c908c219e4053..9a431717d9fb9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -174,4 +175,10 @@ public int hashCode() {
     public static String invalidInputTypeMessage(InputType inputType) {
         return Strings.format("received invalid input type value [%s]", inputType.toString());
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AlibabaCloudSearchEmbeddingsTaskSettings newSettingsOnly = fromMap(new HashMap<>(newSettings));
+        return of(this, newSettingsOnly, newSettingsOnly.inputType != null ? newSettingsOnly.inputType : this.getInputType());
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java
index 97e7ecd41223d..40c3dee00d6c7 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java
@@ -102,4 +102,10 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash();
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AlibabaCloudSearchRerankTaskSettings updatedSettings = new AlibabaCloudSearchRerankTaskSettings();
+        return of(this, updatedSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java
index 873cdf31fbe9d..0f4ebce920167 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -188,4 +189,10 @@ public int hashCode() {
     public static String invalidInputTypeMessage(InputType inputType) {
         return Strings.format("received invalid input type value [%s]", inputType.toString());
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AlibabaCloudSearchSparseTaskSettings updatedSettings = fromMap(new HashMap<>(newSettings));
+        return of(this, updatedSettings, updatedSettings.getInputType() != null ? updatedSettings.getInputType() : this.inputType);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java
index 9e6328ce1c358..0ca71d47eb1b6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.services.amazonbedrock;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -18,10 +19,10 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.TransportVersions.ML_INFERENCE_AMAZON_BEDROCK_ADDED;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredSecureString;
 import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.ACCESS_KEY_FIELD;
 import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.SECRET_KEY_FIELD;
@@ -75,7 +76,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return ML_INFERENCE_AMAZON_BEDROCK_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
@@ -107,4 +108,9 @@ public boolean equals(Object object) {
     public int hashCode() {
         return Objects.hash(accessKey, secretKey);
     }
+
+    @Override
+    public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+        return fromMap(new HashMap<>(newSecrets));
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java
index 9b066f2a1679a..96dd6d2b3690f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.IOUtils;
@@ -44,8 +45,8 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
-import static org.elasticsearch.TransportVersions.ML_INFERENCE_AMAZON_BEDROCK_ADDED;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
@@ -264,7 +265,12 @@ private static AmazonBedrockModel createModel(
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return ML_INFERENCE_AMAZON_BEDROCK_ADDED;
+        return TransportVersions.V_8_15_0;
+    }
+
+    @Override
+    public Set<TaskType> supportedStreamingTasks() {
+        return COMPLETION_ONLY;
     }
 
     /**
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java
index b572df4f1ee05..f11f5818f635c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.services.amazonbedrock;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -24,7 +25,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.TransportVersions.ML_INFERENCE_AMAZON_BEDROCK_ADDED;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredEnum;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString;
 import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.MODEL_FIELD;
@@ -101,7 +101,7 @@ protected AmazonBedrockServiceSettings(
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return ML_INFERENCE_AMAZON_BEDROCK_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     public String region() {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java
index 13787ed8cb6a4..65c227bbbd412 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.services.amazonbedrock.completion;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -17,10 +18,10 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.TransportVersions.ML_INFERENCE_AMAZON_BEDROCK_ADDED;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalDoubleInRange;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger;
 import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.MAX_NEW_TOKENS_FIELD;
@@ -140,7 +141,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return ML_INFERENCE_AMAZON_BEDROCK_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
@@ -192,4 +193,12 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(temperature, topP, topK, maxNewTokens);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AmazonBedrockChatCompletionRequestTaskSettings requestSettings = AmazonBedrockChatCompletionRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, requestSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java
index 8d6f026e12ac5..c925053c38116 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java
@@ -198,7 +198,7 @@ protected void doChunkedInfer(
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_ANTHROPIC_INTEGRATION_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java
index bb2c027127371..305502d624826 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -59,6 +60,11 @@ private static AnthropicChatCompletionTaskSettings fromPersistedMap(Map<String,
         return new AnthropicChatCompletionTaskSettings(commonFields);
     }
 
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        return fromRequestMap(new HashMap<>(newSettings));
+    }
+
     private record CommonFields(int maxTokens, Double temperature, Double topP, Integer topK) {}
 
     private static CommonFields fromMap(Map<String, Object> map, ValidationException validationException) {
@@ -161,7 +167,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_ANTHROPIC_INTEGRATION_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
index ba36febc3c162..5525fff6b1a7c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
@@ -214,7 +214,7 @@ public String name() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_AI_STUDIO;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettings.java
index 2f8422be5ed90..ecc8f7fe8e920 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettings.java
@@ -80,7 +80,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_AI_STUDIO;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java
index b8e33bac410fe..544c52f59a3c4 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsTaskSettings;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -178,6 +179,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    @Override
+    public String toString() {
+        return "AzureAiStudioChatCompletionTaskSettings{"
+            + "temperature="
+            + temperature
+            + ", topP="
+            + topP
+            + ", doSample="
+            + doSample
+            + ", maxNewTokens="
+            + maxNewTokens
+            + '}';
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -194,4 +209,11 @@ public int hashCode() {
         return Objects.hash(temperature, topP, doSample, maxNewTokens);
     }
 
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AzureAiStudioChatCompletionRequestTaskSettings requestSettings = AzureAiStudioChatCompletionRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, requestSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettings.java
index 8977ba8b12836..6ae1f773853d1 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettings.java
@@ -166,7 +166,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_AI_STUDIO;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java
index bdb6ae74e5ab3..340ee95cd7b0c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsTaskSettings;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -111,4 +112,12 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hashCode(user);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AzureAiStudioEmbeddingsRequestTaskSettings requestSettings = AzureAiStudioEmbeddingsRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return AzureAiStudioEmbeddingsTaskSettings.of(this, requestSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
index 06217e8079b06..a2bd4f6175989 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -125,4 +126,9 @@ public boolean equals(Object object) {
     public int hashCode() {
         return Objects.hash(entraId, apiKey);
     }
+
+    @Override
+    public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+        return AzureOpenAiSecretSettings.fromMap(new HashMap<>(newSecrets));
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
index e1c230b98a2f7..cd657113d7b61 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
@@ -324,7 +324,7 @@ private AzureOpenAiEmbeddingsModel updateModelWithEmbeddingDetails(AzureOpenAiEm
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java
index 16f58574a8068..eb7eed2cedb41 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java
@@ -168,7 +168,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_OPENAI_COMPLETIONS;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java
index de0a0897a93c5..7f90b866a2eab 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -87,7 +88,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_OPENAI_COMPLETIONS;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
@@ -107,4 +108,12 @@ public boolean equals(Object object) {
     public int hashCode() {
         return Objects.hash(user);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AzureOpenAiCompletionRequestTaskSettings updatedSettings = AzureOpenAiCompletionRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, updatedSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
index 941a4bdeeb41a..9e8ca478987ec 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
@@ -185,7 +185,7 @@ public AzureOpenAiEmbeddingsServiceSettings(StreamInput in) throws IOException {
         maxInputTokens = in.readOptionalVInt();
         similarity = in.readOptionalEnum(SimilarityMeasure.class);
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings = new RateLimitSettings(in);
         } else {
             rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS;
@@ -303,7 +303,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalVInt(maxInputTokens);
         out.writeOptionalEnum(SimilarityMeasure.translateSimilarity(similarity, out.getTransportVersion()));
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java
index 28ccade0a06b0..4157d7748d789 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -116,4 +117,12 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(user);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        AzureOpenAiEmbeddingsRequestTaskSettings requestSettings = AzureOpenAiEmbeddingsRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, requestSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
index 9538588700e75..ce0fa0a885a20 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
@@ -325,7 +325,7 @@ static SimilarityMeasure defaultSimilarity() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java
index 7f28459e9b8be..a1943b339a561 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java
@@ -124,7 +124,7 @@ public CohereServiceSettings(StreamInput in) throws IOException {
         maxInputTokens = in.readOptionalVInt();
         modelId = in.readOptionalString();
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings = new RateLimitSettings(in);
         } else {
             rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS;
@@ -219,7 +219,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalVInt(maxInputTokens);
         out.writeOptionalString(modelId);
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java
index ba9e81b461f9f..be241f3aaa7fc 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java
@@ -114,7 +114,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_COHERE_COMPLETION_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java
index 34d37d0003adf..b789d1578290a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java
@@ -22,6 +22,7 @@
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -204,4 +205,10 @@ public int hashCode() {
     public static String invalidInputTypeMessage(InputType inputType) {
         return Strings.format("received invalid input type value [%s]", inputType.toString());
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        CohereEmbeddingsTaskSettings updatedSettings = CohereEmbeddingsTaskSettings.fromMap(new HashMap<>(newSettings));
+        return of(this, updatedSettings, updatedSettings.inputType != null ? updatedSettings.inputType : this.inputType);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java
index 1132dff34ed6e..a3d2483a068e2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java
@@ -101,7 +101,7 @@ public CohereRerankServiceSettings(StreamInput in) throws IOException {
 
         this.modelId = in.readOptionalString();
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             this.rateLimitSettings = new RateLimitSettings(in);
         } else {
             this.rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS;
@@ -172,7 +172,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
         out.writeOptionalString(modelId);
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java
index f5893c825efcf..479000f840502 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -186,4 +187,9 @@ public Integer getMaxChunksPerDoc() {
         return maxChunksPerDoc;
     }
 
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        CohereRerankTaskSettings updatedSettings = CohereRerankTaskSettings.fromMap(new HashMap<>(newSettings));
+        return CohereRerankTaskSettings.of(this, updatedSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java
index 881e2e82b766a..cd0c33082cb30 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java
@@ -15,13 +15,16 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.client.internal.OriginSettingClient;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.InferenceService;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction;
@@ -38,7 +41,6 @@
 import java.io.IOException;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.function.Consumer;
 
@@ -49,14 +51,21 @@ public abstract class BaseElasticsearchInternalService implements InferenceServi
 
     protected final OriginSettingClient client;
     protected final ExecutorService inferenceExecutor;
-    protected final Consumer<ActionListener<Set<String>>> platformArch;
+    protected final Consumer<ActionListener<PreferredModelVariant>> preferredModelVariantFn;
+    private final ClusterService clusterService;
+
+    public enum PreferredModelVariant {
+        LINUX_X86_OPTIMIZED,
+        PLATFORM_AGNOSTIC
+    };
 
     private static final Logger logger = LogManager.getLogger(BaseElasticsearchInternalService.class);
 
     public BaseElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) {
         this.client = new OriginSettingClient(context.client(), ClientHelper.INFERENCE_ORIGIN);
         this.inferenceExecutor = context.threadPool().executor(InferencePlugin.UTILITY_THREAD_POOL_NAME);
-        this.platformArch = this::platformArchitecture;
+        this.preferredModelVariantFn = this::preferredVariantFromPlatformArchitecture;
+        this.clusterService = context.clusterService();
     }
 
     // For testing.
@@ -66,11 +75,12 @@ public BaseElasticsearchInternalService(InferenceServiceExtension.InferenceServi
     // service package.
     public BaseElasticsearchInternalService(
         InferenceServiceExtension.InferenceServiceFactoryContext context,
-        Consumer<ActionListener<Set<String>>> platformArchFn
+        Consumer<ActionListener<PreferredModelVariant>> preferredModelVariantFn
     ) {
         this.client = new OriginSettingClient(context.client(), ClientHelper.INFERENCE_ORIGIN);
         this.inferenceExecutor = context.threadPool().executor(InferencePlugin.UTILITY_THREAD_POOL_NAME);
-        this.platformArch = platformArchFn;
+        this.preferredModelVariantFn = preferredModelVariantFn;
+        this.clusterService = context.clusterService();
     }
 
     /**
@@ -89,6 +99,12 @@ public void start(Model model, ActionListener<Boolean> finalListener) {
                 return;
             }
 
+            if (esModel.usesExistingDeployment()) {
+                // don't start a deployment
+                finalListener.onResponse(Boolean.TRUE);
+                return;
+            }
+
             SubscribableListener.<Boolean>newForked(forkedListener -> { isBuiltinModelPut(model, forkedListener); })
                 .<Boolean>andThen((l, modelConfigExists) -> {
                     if (modelConfigExists == false) {
@@ -110,14 +126,28 @@ public void start(Model model, ActionListener<Boolean> finalListener) {
     }
 
     @Override
-    public void stop(String inferenceEntityId, ActionListener<Boolean> listener) {
-        var request = new StopTrainedModelDeploymentAction.Request(inferenceEntityId);
-        request.setForce(true);
-        client.execute(
-            StopTrainedModelDeploymentAction.INSTANCE,
-            request,
-            listener.delegateFailureAndWrap((delegatedResponseListener, response) -> delegatedResponseListener.onResponse(Boolean.TRUE))
-        );
+    public void stop(UnparsedModel unparsedModel, ActionListener<Boolean> listener) {
+
+        var model = parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings());
+        if (model instanceof ElasticsearchInternalModel esModel) {
+
+            var serviceSettings = esModel.getServiceSettings();
+            if (serviceSettings.getDeploymentId() != null) {
+                // configured with an existing deployment so do not stop it
+                listener.onResponse(Boolean.TRUE);
+                return;
+            }
+
+            var request = new StopTrainedModelDeploymentAction.Request(esModel.mlNodeDeploymentId());
+            request.setForce(true);
+            client.execute(
+                StopTrainedModelDeploymentAction.INSTANCE,
+                request,
+                listener.delegateFailureAndWrap((delegatedResponseListener, response) -> delegatedResponseListener.onResponse(Boolean.TRUE))
+            );
+        } else {
+            listener.onFailure(notElasticsearchModelException(model));
+        }
     }
 
     protected static IllegalStateException notElasticsearchModelException(Model model) {
@@ -206,31 +236,36 @@ protected void isBuiltinModelPut(Model model, ActionListener<Boolean> listener)
     public void close() throws IOException {}
 
     public static String selectDefaultModelVariantBasedOnClusterArchitecture(
-        Set<String> modelArchitectures,
-        String linuxX86OptimisedModel,
+        PreferredModelVariant preferredModelVariant,
+        String linuxX86OptimizedModel,
         String platformAgnosticModel
     ) {
         // choose a default model version based on the cluster architecture
-        boolean homogenous = modelArchitectures.size() == 1;
-        if (homogenous && modelArchitectures.iterator().next().equals("linux-x86_64")) {
+        if (PreferredModelVariant.LINUX_X86_OPTIMIZED.equals(preferredModelVariant)) {
             // Use the hardware optimized model
-            return linuxX86OptimisedModel;
+            return linuxX86OptimizedModel;
         } else {
             // default to the platform-agnostic model
             return platformAgnosticModel;
         }
     }
 
-    private void platformArchitecture(ActionListener<Set<String>> platformArchitectureListener) {
+    private void preferredVariantFromPlatformArchitecture(ActionListener<PreferredModelVariant> preferredVariantListener) {
         // Find the cluster platform as the service may need that
         // information when creating the model
         MlPlatformArchitecturesUtil.getMlNodesArchitecturesSet(
-            platformArchitectureListener.delegateFailureAndWrap((delegate, architectures) -> {
-                if (architectures.isEmpty() && clusterIsInElasticCloud()) {
-                    // In Elastic cloud ml nodes run on Linux x86
-                    delegate.onResponse(Set.of("linux-x86_64"));
+            preferredVariantListener.delegateFailureAndWrap((delegate, architectures) -> {
+                if (architectures.isEmpty() && isClusterInElasticCloud()) {
+                    // There are no ml nodes to check the current arch.
+                    // However, in Elastic cloud ml nodes run on Linux x86
+                    delegate.onResponse(PreferredModelVariant.LINUX_X86_OPTIMIZED);
                 } else {
-                    delegate.onResponse(architectures);
+                    boolean homogenous = architectures.size() == 1;
+                    if (homogenous && architectures.iterator().next().equals("linux-x86_64")) {
+                        delegate.onResponse(PreferredModelVariant.LINUX_X86_OPTIMIZED);
+                    } else {
+                        delegate.onResponse(PreferredModelVariant.PLATFORM_AGNOSTIC);
+                    }
                 }
             }),
             client,
@@ -238,9 +273,11 @@ private void platformArchitecture(ActionListener<Set<String>> platformArchitectu
         );
     }
 
-    static boolean clusterIsInElasticCloud() {
-        // use a heuristic to determine if in Elastic cloud.
-        return true; // TODO
+    boolean isClusterInElasticCloud() {
+        // Use the ml lazy node count as a heuristic to determine if in Elastic cloud.
+        // A value > 0 means scaling should be available for ml nodes
+        var maxMlLazyNodes = clusterService.getClusterSettings().get(MachineLearningField.MAX_LAZY_ML_NODES);
+        return maxMlLazyNodes > 0;
     }
 
     public static InferModelAction.Request buildInferenceRequest(
@@ -259,7 +296,7 @@ public static InferModelAction.Request buildInferenceRequest(
         return request;
     }
 
-    protected abstract boolean isDefaultId(String inferenceId);
+    abstract boolean isDefaultId(String inferenceId);
 
     protected void maybeStartDeployment(
         ElasticsearchInternalModel model,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java
index 381c97969e79f..133be5e2b7623 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java
@@ -139,7 +139,7 @@ public CustomElandInternalTextEmbeddingServiceSettings(
 
     public CustomElandInternalTextEmbeddingServiceSettings(StreamInput in) throws IOException {
         super(in);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ELAND_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             dimensions = in.readOptionalVInt();
             similarityMeasure = in.readEnum(SimilarityMeasure.class);
             elementType = in.readEnum(DenseVectorFieldMapper.ElementType.class);
@@ -197,7 +197,7 @@ public String getWriteableName() {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ELAND_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeOptionalVInt(dimensions);
             out.writeEnum(similarityMeasure);
             out.writeEnum(elementType);
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java
index 70d787152121f..a0be1661b860d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -87,7 +88,11 @@ public CustomElandRerankTaskSettings(StreamInput in) throws IOException {
     }
 
     public CustomElandRerankTaskSettings(@Nullable Boolean doReturnDocuments) {
-        this.returnDocuments = doReturnDocuments;
+        if (doReturnDocuments == null) {
+            this.returnDocuments = true;
+        } else {
+            this.returnDocuments = doReturnDocuments;
+        }
     }
 
     @Override
@@ -136,4 +141,10 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(returnDocuments);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        CustomElandRerankTaskSettings updatedSettings = CustomElandRerankTaskSettings.fromMap(new HashMap<>(newSettings));
+        return of(this, updatedSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticDeployedModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticDeployedModel.java
new file mode 100644
index 0000000000000..996ef6816025d
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticDeployedModel.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elasticsearch;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.ChunkingSettings;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction;
+import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
+
+public class ElasticDeployedModel extends ElasticsearchInternalModel {
+    public ElasticDeployedModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        ElasticsearchInternalServiceSettings serviceSettings,
+        ChunkingSettings chunkingSettings
+    ) {
+        super(inferenceEntityId, taskType, service, serviceSettings, chunkingSettings);
+    }
+
+    @Override
+    public boolean usesExistingDeployment() {
+        return true;
+    }
+
+    @Override
+    public StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() {
+        throw new IllegalStateException("cannot start model that uses an existing deployment");
+    }
+
+    @Override
+    public ActionListener<CreateTrainedModelAssignmentAction.Response> getCreateTrainedModelAssignmentActionListener(
+        Model model,
+        ActionListener<Boolean> listener
+    ) {
+        throw new IllegalStateException("cannot start model that uses an existing deployment");
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java
index 2aee37312a16e..8b2969c39b7ba 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java
@@ -21,7 +21,7 @@
 
 public abstract class ElasticsearchInternalModel extends Model {
 
-    protected final ElasticsearchInternalServiceSettings internalServiceSettings;
+    protected ElasticsearchInternalServiceSettings internalServiceSettings;
 
     public ElasticsearchInternalModel(
         String inferenceEntityId,
@@ -82,13 +82,25 @@ public abstract ActionListener<CreateTrainedModelAssignmentAction.Response> getC
         ActionListener<Boolean> listener
     );
 
+    public boolean usesExistingDeployment() {
+        return internalServiceSettings.getDeploymentId() != null;
+    }
+
     @Override
     public ElasticsearchInternalServiceSettings getServiceSettings() {
         return (ElasticsearchInternalServiceSettings) super.getServiceSettings();
     }
 
+    public void updateNumAllocations(Integer numAllocations) {
+        this.internalServiceSettings.setNumAllocations(numAllocations);
+    }
+
     @Override
     public String toString() {
         return Strings.toString(this.getConfigurations());
     }
+
+    public String mlNodeDeploymentId() {
+        return internalServiceSettings.getDeploymentId() == null ? getInferenceEntityId() : internalServiceSettings.getDeploymentId();
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
index 9a4201842873e..fec690199d97d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
@@ -27,20 +27,26 @@
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag;
 import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.EmptyConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
 import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
@@ -50,8 +56,10 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -62,7 +70,6 @@
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL_LINUX_X86;
-import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.EMBEDDING_MAX_BATCH_SIZE;
 
 public class ElasticsearchInternalService extends BaseElasticsearchInternalService {
 
@@ -77,7 +84,8 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi
     );
 
     public static final int EMBEDDING_MAX_BATCH_SIZE = 10;
-    public static final String DEFAULT_ELSER_ID = ".elser-2";
+    public static final String DEFAULT_ELSER_ID = ".elser-2-elasticsearch";
+    public static final String DEFAULT_E5_ID = ".multilingual-e5-small-elasticsearch";
 
     private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class);
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(ElasticsearchInternalService.class);
@@ -89,7 +97,7 @@ public ElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFa
     // for testing
     ElasticsearchInternalService(
         InferenceServiceExtension.InferenceServiceFactoryContext context,
-        Consumer<ActionListener<Set<String>>> platformArch
+        Consumer<ActionListener<PreferredModelVariant>> platformArch
     ) {
         super(context, platformArch);
     }
@@ -135,7 +143,12 @@ public void parseRequestConfig(
             throwIfNotEmptyMap(config, name());
 
             String modelId = (String) serviceSettingsMap.get(ElasticsearchInternalServiceSettings.MODEL_ID);
-            if (modelId == null) {
+            String deploymentId = (String) serviceSettingsMap.get(ElasticsearchInternalServiceSettings.DEPLOYMENT_ID);
+            if (deploymentId != null) {
+                validateAgainstDeployment(modelId, deploymentId, taskType, modelListener.delegateFailureAndWrap((l, settings) -> {
+                    l.onResponse(new ElasticDeployedModel(inferenceEntityId, taskType, NAME, settings.build(), chunkingSettings));
+                }));
+            } else if (modelId == null) {
                 if (OLD_ELSER_SERVICE_NAME.equals(serviceName)) {
                     // TODO complete deprecation of null model ID
                     // throw new ValidationException().addValidationError("Error parsing request config, model id is missing");
@@ -145,14 +158,15 @@ public void parseRequestConfig(
                         "Putting elasticsearch service inference endpoints (including elser service) without a model_id field is"
                             + " deprecated and will be removed in a future release. Please specify a model_id field."
                     );
-                    platformArch.accept(
+                    preferredModelVariantFn.accept(
                         modelListener.delegateFailureAndWrap(
-                            (delegate, arch) -> elserCase(
+                            (delegate, preferredModelVariant) -> elserCase(
                                 inferenceEntityId,
                                 taskType,
                                 config,
-                                arch,
+                                preferredModelVariant,
                                 serviceSettingsMap,
+                                true,
                                 chunkingSettings,
                                 modelListener
                             )
@@ -162,13 +176,13 @@ public void parseRequestConfig(
                     throw new IllegalArgumentException("Error parsing service settings, model_id must be provided");
                 }
             } else if (MULTILINGUAL_E5_SMALL_VALID_IDS.contains(modelId)) {
-                platformArch.accept(
+                preferredModelVariantFn.accept(
                     modelListener.delegateFailureAndWrap(
-                        (delegate, arch) -> e5Case(
+                        (delegate, preferredModelVariant) -> e5Case(
                             inferenceEntityId,
                             taskType,
                             config,
-                            arch,
+                            preferredModelVariant,
                             serviceSettingsMap,
                             chunkingSettings,
                             modelListener
@@ -176,14 +190,15 @@ public void parseRequestConfig(
                     )
                 );
             } else if (ElserModels.isValidModel(modelId)) {
-                platformArch.accept(
+                preferredModelVariantFn.accept(
                     modelListener.delegateFailureAndWrap(
-                        (delegate, arch) -> elserCase(
+                        (delegate, preferredModelVariant) -> elserCase(
                             inferenceEntityId,
                             taskType,
                             config,
-                            arch,
+                            preferredModelVariant,
                             serviceSettingsMap,
+                            OLD_ELSER_SERVICE_NAME.equals(serviceName),
                             chunkingSettings,
                             modelListener
                         )
@@ -216,6 +231,8 @@ private void customElandCase(
                         + "]. You may need to load it into the cluster using eland."
                 );
             } else {
+                throwIfUnsupportedTaskType(modelId, taskType, response.getResources().results().get(0).getInferenceConfig());
+
                 var model = createCustomElandModel(
                     inferenceEntityId,
                     taskType,
@@ -286,7 +303,7 @@ private void e5Case(
         String inferenceEntityId,
         TaskType taskType,
         Map<String, Object> config,
-        Set<String> platformArchitectures,
+        PreferredModelVariant preferredModelVariant,
         Map<String, Object> serviceSettingsMap,
         ChunkingSettings chunkingSettings,
         ActionListener<Model> modelListener
@@ -296,12 +313,12 @@ private void e5Case(
         if (esServiceSettingsBuilder.getModelId() == null) {
             esServiceSettingsBuilder.setModelId(
                 selectDefaultModelVariantBasedOnClusterArchitecture(
-                    platformArchitectures,
+                    preferredModelVariant,
                     MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86,
                     MULTILINGUAL_E5_SMALL_MODEL_ID
                 )
             );
-        } else if (modelVariantValidForArchitecture(platformArchitectures, esServiceSettingsBuilder.getModelId()) == false) {
+        } else if (modelVariantValidForArchitecture(preferredModelVariant, esServiceSettingsBuilder.getModelId()) == false) {
             throw new IllegalArgumentException(
                 "Error parsing request config, model id does not match any models available on this platform. Was ["
                     + esServiceSettingsBuilder.getModelId()
@@ -323,14 +340,14 @@ private void e5Case(
         );
     }
 
-    static boolean modelVariantValidForArchitecture(Set<String> platformArchitectures, String modelId) {
+    static boolean modelVariantValidForArchitecture(PreferredModelVariant modelVariant, String modelId) {
         if (modelId.equals(MULTILINGUAL_E5_SMALL_MODEL_ID)) {
             // platform agnostic model is always compatible
             return true;
         }
         return modelId.equals(
             selectDefaultModelVariantBasedOnClusterArchitecture(
-                platformArchitectures,
+                modelVariant,
                 MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86,
                 MULTILINGUAL_E5_SMALL_MODEL_ID
             )
@@ -341,14 +358,15 @@ private void elserCase(
         String inferenceEntityId,
         TaskType taskType,
         Map<String, Object> config,
-        Set<String> platformArchitectures,
+        PreferredModelVariant preferredModelVariant,
         Map<String, Object> serviceSettingsMap,
+        boolean isElserService,
         ChunkingSettings chunkingSettings,
         ActionListener<Model> modelListener
     ) {
         var esServiceSettingsBuilder = ElasticsearchInternalServiceSettings.fromRequestMap(serviceSettingsMap);
         final String defaultModelId = selectDefaultModelVariantBasedOnClusterArchitecture(
-            platformArchitectures,
+            preferredModelVariant,
             ELSER_V2_MODEL_LINUX_X86,
             ELSER_V2_MODEL
         );
@@ -373,21 +391,15 @@ private void elserCase(
             }
         }
 
-        DEPRECATION_LOGGER.warn(
-            DeprecationCategory.API,
-            "inference_api_elser_service",
-            "The [{}] service is deprecated and will be removed in a future release. Use the [{}] service instead, with"
-                + " [model_id] set to [{}] in the [service_settings]",
-            OLD_ELSER_SERVICE_NAME,
-            ElasticsearchInternalService.NAME,
-            defaultModelId
-        );
-
-        if (modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic(platformArchitectures, esServiceSettingsBuilder.getModelId())) {
-            throw new IllegalArgumentException(
-                "Error parsing request config, model id does not match any models available on this platform. Was ["
-                    + esServiceSettingsBuilder.getModelId()
-                    + "]"
+        if (isElserService) {
+            DEPRECATION_LOGGER.warn(
+                DeprecationCategory.API,
+                "inference_api_elser_service",
+                "The [{}] service is deprecated and will be removed in a future release. Use the [{}] service instead, with"
+                    + " [model_id] set to [{}] in the [service_settings]",
+                OLD_ELSER_SERVICE_NAME,
+                ElasticsearchInternalService.NAME,
+                defaultModelId
             );
         }
 
@@ -406,19 +418,6 @@ private void elserCase(
         );
     }
 
-    private static boolean modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic(
-        Set<String> platformArchitectures,
-        String modelId
-    ) {
-        return modelId.equals(
-            selectDefaultModelVariantBasedOnClusterArchitecture(
-                platformArchitectures,
-                MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86,
-                MULTILINGUAL_E5_SMALL_MODEL_ID
-            )
-        );
-    }
-
     @Override
     public Model parsePersistedConfigWithSecrets(
         String inferenceEntityId,
@@ -554,7 +553,7 @@ public void inferTextEmbedding(
         ActionListener<InferenceServiceResults> listener
     ) {
         var request = buildInferenceRequest(
-            model.getConfigurations().getInferenceEntityId(),
+            model.mlNodeDeploymentId(),
             TextEmbeddingConfigUpdate.EMPTY_INSTANCE,
             inputs,
             inputType,
@@ -579,13 +578,7 @@ public void inferSparseEmbedding(
         TimeValue timeout,
         ActionListener<InferenceServiceResults> listener
     ) {
-        var request = buildInferenceRequest(
-            model.getConfigurations().getInferenceEntityId(),
-            TextExpansionConfigUpdate.EMPTY_UPDATE,
-            inputs,
-            inputType,
-            timeout
-        );
+        var request = buildInferenceRequest(model.mlNodeDeploymentId(), TextExpansionConfigUpdate.EMPTY_UPDATE, inputs, inputType, timeout);
 
         ActionListener<InferModelAction.Response> mlResultsListener = listener.delegateFailureAndWrap(
             (l, inferenceResult) -> l.onResponse(SparseEmbeddingResults.of(inferenceResult.getInferenceResults()))
@@ -607,13 +600,7 @@ public void inferRerank(
         Map<String, Object> requestTaskSettings,
         ActionListener<InferenceServiceResults> listener
     ) {
-        var request = buildInferenceRequest(
-            model.getConfigurations().getInferenceEntityId(),
-            new TextSimilarityConfigUpdate(query),
-            inputs,
-            inputType,
-            timeout
-        );
+        var request = buildInferenceRequest(model.mlNodeDeploymentId(), new TextSimilarityConfigUpdate(query), inputs, inputType, timeout);
 
         var modelSettings = (CustomElandRerankTaskSettings) model.getTaskSettings();
         var requestSettings = CustomElandRerankTaskSettings.fromMap(requestTaskSettings);
@@ -682,7 +669,7 @@ public void chunkedInfer(
 
             for (var batch : batchedRequests) {
                 var inferenceRequest = buildInferenceRequest(
-                    model.getConfigurations().getInferenceEntityId(),
+                    esModel.mlNodeDeploymentId(),
                     EmptyConfigUpdate.INSTANCE,
                     batch.batch().inputs(),
                     inputType,
@@ -793,42 +780,107 @@ private RankedDocsResults textSimilarityResultsToRankedDocs(
         return new RankedDocsResults(rankings);
     }
 
+    public List<DefaultConfigId> defaultConfigIds() {
+        return List.of(
+            new DefaultConfigId(DEFAULT_ELSER_ID, TaskType.SPARSE_EMBEDDING, this),
+            new DefaultConfigId(DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, this)
+        );
+    }
+
     @Override
-    public List<UnparsedModel> defaultConfigs() {
-        // TODO Chunking settings
-        Map<String, Object> elserSettings = Map.of(
-            ModelConfigurations.SERVICE_SETTINGS,
-            Map.of(
-                ElasticsearchInternalServiceSettings.MODEL_ID,
-                ElserModels.ELSER_V2_MODEL,  // TODO pick model depending on platform
-                ElasticsearchInternalServiceSettings.NUM_THREADS,
-                1,
-                ElasticsearchInternalServiceSettings.ADAPTIVE_ALLOCATIONS,
-                Map.of(
-                    "enabled",
-                    Boolean.TRUE,
-                    "min_number_of_allocations",
-                    1,
-                    "max_number_of_allocations",
-                    8   // no max?
-                )
-            )
+    public void updateModelsWithDynamicFields(List<Model> models, ActionListener<List<Model>> listener) {
+
+        if (models.isEmpty()) {
+            listener.onResponse(models);
+            return;
+        }
+
+        var modelsByDeploymentIds = new HashMap<String, ElasticsearchInternalModel>();
+        for (var model : models) {
+            assert model instanceof ElasticsearchInternalModel;
+
+            if (model instanceof ElasticsearchInternalModel esModel) {
+                modelsByDeploymentIds.put(esModel.mlNodeDeploymentId(), esModel);
+            } else {
+                listener.onFailure(
+                    new ElasticsearchStatusException(
+                        "Cannot update model [{}] as it is not an Elasticsearch service model",
+                        RestStatus.INTERNAL_SERVER_ERROR,
+                        model.getInferenceEntityId()
+                    )
+                );
+                return;
+            }
+        }
+
+        String deploymentIds = String.join(",", modelsByDeploymentIds.keySet());
+        client.execute(
+            GetDeploymentStatsAction.INSTANCE,
+            new GetDeploymentStatsAction.Request(deploymentIds),
+            ActionListener.wrap(stats -> {
+                for (var deploymentStats : stats.getStats().results()) {
+                    var model = modelsByDeploymentIds.get(deploymentStats.getDeploymentId());
+                    model.updateNumAllocations(deploymentStats.getNumberOfAllocations());
+                }
+                listener.onResponse(new ArrayList<>(modelsByDeploymentIds.values()));
+            }, e -> {
+                logger.warn("Get deployment stats failed, cannot update the endpoint's number of allocations", e);
+                // continue with the original response
+                listener.onResponse(models);
+            })
         );
+    }
 
-        return List.of(
-            new UnparsedModel(
-                DEFAULT_ELSER_ID,
-                TaskType.SPARSE_EMBEDDING,
-                NAME,
-                elserSettings,
-                Map.of() // no secrets
-            )
+    public void defaultConfigs(ActionListener<List<Model>> defaultsListener) {
+        preferredModelVariantFn.accept(defaultsListener.delegateFailureAndWrap((delegate, preferredModelVariant) -> {
+            if (PreferredModelVariant.LINUX_X86_OPTIMIZED.equals(preferredModelVariant)) {
+                defaultsListener.onResponse(defaultConfigsLinuxOptimized());
+            } else {
+                defaultsListener.onResponse(defaultConfigsPlatfromAgnostic());
+            }
+        }));
+    }
+
+    private List<Model> defaultConfigsLinuxOptimized() {
+        return defaultConfigs(true);
+    }
+
+    private List<Model> defaultConfigsPlatfromAgnostic() {
+        return defaultConfigs(false);
+    }
+
+    private List<Model> defaultConfigs(boolean useLinuxOptimizedModel) {
+        var defaultElser = new ElserInternalModel(
+            DEFAULT_ELSER_ID,
+            TaskType.SPARSE_EMBEDDING,
+            NAME,
+            new ElserInternalServiceSettings(
+                null,
+                1,
+                useLinuxOptimizedModel ? ELSER_V2_MODEL_LINUX_X86 : ELSER_V2_MODEL,
+                new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32)
+            ),
+            ElserMlNodeTaskSettings.DEFAULT,
+            null // default chunking settings
         );
+        var defaultE5 = new MultilingualE5SmallModel(
+            DEFAULT_E5_ID,
+            TaskType.TEXT_EMBEDDING,
+            NAME,
+            new MultilingualE5SmallInternalServiceSettings(
+                null,
+                1,
+                useLinuxOptimizedModel ? MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 : MULTILINGUAL_E5_SMALL_MODEL_ID,
+                new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32)
+            ),
+            null // default chunking settings
+        );
+        return List.of(defaultElser, defaultE5);
     }
 
     @Override
-    protected boolean isDefaultId(String inferenceId) {
-        return DEFAULT_ELSER_ID.equals(inferenceId);
+    boolean isDefaultId(String inferenceId) {
+        return DEFAULT_ELSER_ID.equals(inferenceId) || DEFAULT_E5_ID.equals(inferenceId);
     }
 
     static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSettings(
@@ -847,4 +899,109 @@ static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSetting
             );
         };
     }
+
+    private void validateAgainstDeployment(
+        String modelId,
+        String deploymentId,
+        TaskType taskType,
+        ActionListener<ElasticsearchInternalServiceSettings.Builder> listener
+    ) {
+        getDeployment(deploymentId, listener.delegateFailureAndWrap((l, response) -> {
+            if (response.isPresent()) {
+                if (modelId != null && modelId.equals(response.get().getModelId()) == false) {
+                    listener.onFailure(
+                        new ElasticsearchStatusException(
+                            "Deployment [{}] uses model [{}] which does not match the model [{}] in the request.",
+                            RestStatus.BAD_REQUEST,
+                            deploymentId,
+                            response.get().getModelId(),
+                            modelId
+                        )
+                    );
+                    return;
+                }
+
+                var updatedSettings = new ElasticsearchInternalServiceSettings.Builder().setNumAllocations(
+                    response.get().getNumberOfAllocations()
+                )
+                    .setNumThreads(response.get().getThreadsPerAllocation())
+                    .setAdaptiveAllocationsSettings(response.get().getAdaptiveAllocationsSettings())
+                    .setDeploymentId(deploymentId)
+                    .setModelId(response.get().getModelId());
+
+                checkTaskTypeForMlNodeModel(response.get().getModelId(), taskType, l.delegateFailureAndWrap((l2, compatibleTaskType) -> {
+                    l2.onResponse(updatedSettings);
+                }));
+            } else {
+                listener.onFailure(new ElasticsearchStatusException("Cannot find deployment [{}]", RestStatus.NOT_FOUND, deploymentId));
+            }
+        }));
+    }
+
+    private void getDeployment(String deploymentId, ActionListener<Optional<AssignmentStats>> listener) {
+        client.execute(
+            GetDeploymentStatsAction.INSTANCE,
+            new GetDeploymentStatsAction.Request(deploymentId),
+            listener.delegateFailureAndWrap((l, response) -> {
+                l.onResponse(
+                    response.getStats()
+                        .results()
+                        .stream()
+                        .filter(s -> s.getDeploymentId() != null && s.getDeploymentId().equals(deploymentId))
+                        .findFirst()
+                );
+            })
+        );
+    }
+
+    private void checkTaskTypeForMlNodeModel(String modelId, TaskType taskType, ActionListener<Boolean> listener) {
+        client.execute(
+            GetTrainedModelsAction.INSTANCE,
+            new GetTrainedModelsAction.Request(modelId),
+            listener.delegateFailureAndWrap((l, response) -> {
+                if (response.getResources().results().isEmpty()) {
+                    l.onFailure(new IllegalStateException("this shouldn't happen"));
+                    return;
+                }
+
+                var inferenceConfig = response.getResources().results().get(0).getInferenceConfig();
+                throwIfUnsupportedTaskType(modelId, taskType, inferenceConfig);
+                l.onResponse(Boolean.TRUE);
+            })
+        );
+    }
+
+    static void throwIfUnsupportedTaskType(String modelId, TaskType taskType, InferenceConfig inferenceConfig) {
+        var deploymentTaskType = inferenceConfigToTaskType(inferenceConfig);
+        if (deploymentTaskType == null) {
+            throw new ElasticsearchStatusException(
+                "Deployed model [{}] has type [{}] which does not map to any supported task types",
+                RestStatus.BAD_REQUEST,
+                modelId,
+                inferenceConfig.getWriteableName()
+            );
+        }
+        if (deploymentTaskType != taskType) {
+            throw new ElasticsearchStatusException(
+                "Deployed model [{}] with type [{}] does not match the requested task type [{}]",
+                RestStatus.BAD_REQUEST,
+                modelId,
+                inferenceConfig.getWriteableName(),
+                taskType
+            );
+        }
+
+    }
+
+    static TaskType inferenceConfigToTaskType(InferenceConfig config) {
+        if (config instanceof TextExpansionConfig) {
+            return TaskType.SPARSE_EMBEDDING;
+        } else if (config instanceof TextEmbeddingConfig) {
+            return TaskType.TEXT_EMBEDDING;
+        } else if (config instanceof TextSimilarityConfig) {
+            return TaskType.RERANK;
+        } else {
+            return null;
+        }
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java
index f8b5837ef387e..962c939146ef2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java
@@ -36,12 +36,14 @@ public class ElasticsearchInternalServiceSettings implements ServiceSettings {
     public static final String NUM_ALLOCATIONS = "num_allocations";
     public static final String NUM_THREADS = "num_threads";
     public static final String MODEL_ID = "model_id";
+    public static final String DEPLOYMENT_ID = "deployment_id";
     public static final String ADAPTIVE_ALLOCATIONS = "adaptive_allocations";
 
-    private final Integer numAllocations;
+    private Integer numAllocations;
     private final int numThreads;
     private final String modelId;
     private final AdaptiveAllocationsSettings adaptiveAllocationsSettings;
+    private final String deploymentId;
 
     public static ElasticsearchInternalServiceSettings fromPersistedMap(Map<String, Object> map) {
         return fromRequestMap(map).build();
@@ -95,12 +97,15 @@ protected static ElasticsearchInternalServiceSettings.Builder fromMap(
             );
         }
 
+        String deploymentId = extractOptionalString(map, DEPLOYMENT_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
+
         // if an error occurred while parsing, we'll set these to an invalid value, so we don't accidentally get a
         // null pointer when doing unboxing
         return new ElasticsearchInternalServiceSettings.Builder().setNumAllocations(numAllocations)
             .setNumThreads(Objects.requireNonNullElse(numThreads, FAILED_INT_PARSE_VALUE))
             .setModelId(modelId)
-            .setAdaptiveAllocationsSettings(adaptiveAllocationsSettings);
+            .setAdaptiveAllocationsSettings(adaptiveAllocationsSettings)
+            .setDeploymentId(deploymentId);
     }
 
     public ElasticsearchInternalServiceSettings(
@@ -113,6 +118,21 @@ public ElasticsearchInternalServiceSettings(
         this.numThreads = numThreads;
         this.modelId = Objects.requireNonNull(modelId);
         this.adaptiveAllocationsSettings = adaptiveAllocationsSettings;
+        this.deploymentId = null;
+    }
+
+    public ElasticsearchInternalServiceSettings(
+        Integer numAllocations,
+        int numThreads,
+        String modelId,
+        AdaptiveAllocationsSettings adaptiveAllocationsSettings,
+        String deploymentId
+    ) {
+        this.numAllocations = numAllocations;
+        this.numThreads = numThreads;
+        this.modelId = Objects.requireNonNull(modelId);
+        this.adaptiveAllocationsSettings = adaptiveAllocationsSettings;
+        this.deploymentId = deploymentId;
     }
 
     protected ElasticsearchInternalServiceSettings(ElasticsearchInternalServiceSettings other) {
@@ -120,6 +140,20 @@ protected ElasticsearchInternalServiceSettings(ElasticsearchInternalServiceSetti
         this.numThreads = other.numThreads;
         this.modelId = other.modelId;
         this.adaptiveAllocationsSettings = other.adaptiveAllocationsSettings;
+        this.deploymentId = other.deploymentId;
+    }
+
+    /**
+     * Copy constructor with the ability to set the number of allocations. Used for Update API.
+     * @param other the existing settings
+     * @param numAllocations the new number of allocations
+     */
+    public ElasticsearchInternalServiceSettings(ElasticsearchInternalServiceSettings other, int numAllocations) {
+        this.numAllocations = numAllocations;
+        this.numThreads = other.numThreads;
+        this.modelId = other.modelId;
+        this.adaptiveAllocationsSettings = other.adaptiveAllocationsSettings;
+        this.deploymentId = other.deploymentId;
     }
 
     public ElasticsearchInternalServiceSettings(StreamInput in) throws IOException {
@@ -133,6 +167,13 @@ public ElasticsearchInternalServiceSettings(StreamInput in) throws IOException {
         this.adaptiveAllocationsSettings = in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)
             ? in.readOptionalWriteable(AdaptiveAllocationsSettings::new)
             : null;
+        this.deploymentId = in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT)
+            ? in.readOptionalString()
+            : null;
+    }
+
+    public void setNumAllocations(Integer numAllocations) {
+        this.numAllocations = numAllocations;
     }
 
     @Override
@@ -147,6 +188,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) {
             out.writeOptionalWriteable(getAdaptiveAllocationsSettings());
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT)) {
+            out.writeOptionalString(deploymentId);
+        }
     }
 
     @Override
@@ -154,6 +198,10 @@ public String modelId() {
         return modelId;
     }
 
+    public String deloymentId() {
+        return modelId;
+    }
+
     public Integer getNumAllocations() {
         return numAllocations;
     }
@@ -166,6 +214,10 @@ public AdaptiveAllocationsSettings getAdaptiveAllocationsSettings() {
         return adaptiveAllocationsSettings;
     }
 
+    public String getDeploymentId() {
+        return deploymentId;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -183,6 +235,9 @@ protected void addInternalSettingsToXContent(XContentBuilder builder, Params par
         if (adaptiveAllocationsSettings != null) {
             builder.field(ADAPTIVE_ALLOCATIONS, adaptiveAllocationsSettings);
         }
+        if (deploymentId != null) {
+            builder.field(DEPLOYMENT_ID, deploymentId);
+        }
     }
 
     @Override
@@ -205,9 +260,10 @@ public static class Builder {
         private int numThreads;
         private String modelId;
         private AdaptiveAllocationsSettings adaptiveAllocationsSettings;
+        private String deploymentId;
 
         public ElasticsearchInternalServiceSettings build() {
-            return new ElasticsearchInternalServiceSettings(numAllocations, numThreads, modelId, adaptiveAllocationsSettings);
+            return new ElasticsearchInternalServiceSettings(numAllocations, numThreads, modelId, adaptiveAllocationsSettings, deploymentId);
         }
 
         public Builder setNumAllocations(Integer numAllocations) {
@@ -225,6 +281,11 @@ public Builder setModelId(String modelId) {
             return this;
         }
 
+        public Builder setDeploymentId(String deploymentId) {
+            this.deploymentId = deploymentId;
+            return this;
+        }
+
         public Builder setAdaptiveAllocationsSettings(AdaptiveAllocationsSettings adaptiveAllocationsSettings) {
             this.adaptiveAllocationsSettings = adaptiveAllocationsSettings;
             return this;
@@ -254,11 +315,12 @@ public boolean equals(Object o) {
         return Objects.equals(numAllocations, that.numAllocations)
             && numThreads == that.numThreads
             && Objects.equals(modelId, that.modelId)
-            && Objects.equals(adaptiveAllocationsSettings, that.adaptiveAllocationsSettings);
+            && Objects.equals(adaptiveAllocationsSettings, that.adaptiveAllocationsSettings)
+            && Objects.equals(deploymentId, that.deploymentId);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(numAllocations, numThreads, modelId, adaptiveAllocationsSettings);
+        return Objects.hash(numAllocations, numThreads, modelId, adaptiveAllocationsSettings, deploymentId);
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java
index 33696231668a5..3bcaa57827fdb 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.Objects;
 
 public class ElserMlNodeTaskSettings implements TaskSettings {
@@ -65,4 +66,9 @@ public int hashCode() {
         // Return the hash of NAME to make the serialization tests pass
         return Objects.hash(NAME);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        return DEFAULT;
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java
index d817a3bbb73ef..d29095be808b9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java
@@ -7,15 +7,11 @@
 
 package org.elasticsearch.xpack.inference.services.googleaistudio;
 
-import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.ServiceSettings;
-import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
-import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 
-import java.util.Map;
 import java.util.Objects;
 
 public abstract class GoogleAiStudioModel extends Model {
@@ -38,8 +34,6 @@ public GoogleAiStudioModel(GoogleAiStudioModel model, ServiceSettings serviceSet
         rateLimitServiceSettings = model.rateLimitServiceSettings();
     }
 
-    public abstract ExecutableAction accept(GoogleAiStudioActionVisitor creator, Map<String, Object> taskSettings, InputType inputType);
-
     public GoogleAiStudioRateLimitServiceSettings rateLimitServiceSettings() {
         return rateLimitServiceSettings;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
index 798ce4ea5800b..f583caeac8ee3 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
@@ -27,8 +27,11 @@
 import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
 import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
-import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionCreator;
+import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction;
 import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
@@ -42,7 +45,9 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
@@ -208,7 +213,12 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED;
+        return TransportVersions.V_8_15_0;
+    }
+
+    @Override
+    public Set<TaskType> supportedStreamingTasks() {
+        return COMPLETION_ONLY;
     }
 
     @Override
@@ -247,16 +257,32 @@ protected void doInfer(
         TimeValue timeout,
         ActionListener<InferenceServiceResults> listener
     ) {
-        if (model instanceof GoogleAiStudioModel == false) {
+        if (model instanceof GoogleAiStudioCompletionModel completionModel) {
+            var requestManager = new GoogleAiStudioCompletionRequestManager(completionModel, getServiceComponents().threadPool());
+            var docsOnly = DocumentsOnlyInput.of(inputs);
+            var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(
+                completionModel.uri(docsOnly.stream()),
+                "Google AI Studio completion"
+            );
+            var action = new SingleInputSenderExecutableAction(
+                getSender(),
+                requestManager,
+                failedToSendRequestErrorMessage,
+                "Google AI Studio completion"
+            );
+            action.execute(inputs, timeout, listener);
+        } else if (model instanceof GoogleAiStudioEmbeddingsModel embeddingsModel) {
+            var requestManager = new GoogleAiStudioEmbeddingsRequestManager(
+                embeddingsModel,
+                getServiceComponents().truncator(),
+                getServiceComponents().threadPool()
+            );
+            var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(embeddingsModel.uri(), "Google AI Studio embeddings");
+            var action = new SenderExecutableAction(getSender(), requestManager, failedToSendRequestErrorMessage);
+            action.execute(inputs, timeout, listener);
+        } else {
             listener.onFailure(createInvalidModelException(model));
-            return;
         }
-
-        GoogleAiStudioModel googleAiStudioModel = (GoogleAiStudioModel) model;
-        var actionCreator = new GoogleAiStudioActionCreator(getSender(), getServiceComponents());
-
-        var action = googleAiStudioModel.accept(actionCreator, taskSettings, inputType);
-        action.execute(inputs, timeout, listener);
     }
 
     @Override
@@ -270,7 +296,6 @@ protected void doChunkedInfer(
         ActionListener<List<ChunkedInferenceServiceResults>> listener
     ) {
         GoogleAiStudioModel googleAiStudioModel = (GoogleAiStudioModel) model;
-        var actionCreator = new GoogleAiStudioActionCreator(getSender(), getServiceComponents());
 
         List<EmbeddingRequestChunker.BatchRequestAndListener> batchedRequests;
         if (ChunkingSettingsFeatureFlag.isEnabled()) {
@@ -287,10 +312,8 @@ protected void doChunkedInfer(
                 EmbeddingRequestChunker.EmbeddingType.FLOAT
             ).batchRequestsWithListeners(listener);
         }
-
         for (var request : batchedRequests) {
-            var action = googleAiStudioModel.accept(actionCreator, taskSettings, inputType);
-            action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener());
+            doInfer(model, new DocumentsOnlyInput(request.batch().inputs()), taskSettings, inputType, timeout, request.listener());
         }
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
index 8fa2ac0148716..7b793ab37d342 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
@@ -10,13 +10,10 @@
 import org.apache.http.client.utils.URIBuilder;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.EmptyTaskSettings;
-import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.TaskSettings;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
-import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel;
@@ -30,8 +27,6 @@
 
 public class GoogleAiStudioCompletionModel extends GoogleAiStudioModel {
 
-    private URI uri;
-
     public GoogleAiStudioCompletionModel(
         String inferenceEntityId,
         TaskType taskType,
@@ -65,39 +60,20 @@ public GoogleAiStudioCompletionModel(
             new ModelSecrets(secrets),
             serviceSettings
         );
-        try {
-            this.uri = buildUri(serviceSettings.modelId());
-        } catch (URISyntaxException e) {
-            throw new RuntimeException(e);
-        }
     }
 
-    // Should only be used directly for testing
-    GoogleAiStudioCompletionModel(
-        String inferenceEntityId,
-        TaskType taskType,
-        String service,
-        String url,
-        GoogleAiStudioCompletionServiceSettings serviceSettings,
-        TaskSettings taskSettings,
-        @Nullable DefaultSecretSettings secrets
-    ) {
-        super(
-            new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
-            new ModelSecrets(secrets),
-            serviceSettings
-        );
+    public URI uri(boolean streaming) {
         try {
-            this.uri = new URI(url);
+            var api = streaming ? GoogleAiStudioUtils.STREAM_GENERATE_CONTENT_ACTION : GoogleAiStudioUtils.GENERATE_CONTENT_ACTION;
+            return new URIBuilder().setScheme("https")
+                .setHost(GoogleAiStudioUtils.HOST_SUFFIX)
+                .setPathSegments(GoogleAiStudioUtils.V1, GoogleAiStudioUtils.MODELS, format("%s:%s", getServiceSettings().modelId(), api))
+                .build();
         } catch (URISyntaxException e) {
             throw new RuntimeException(e);
         }
     }
 
-    public URI uri() {
-        return uri;
-    }
-
     @Override
     public GoogleAiStudioCompletionServiceSettings getServiceSettings() {
         return (GoogleAiStudioCompletionServiceSettings) super.getServiceSettings();
@@ -108,7 +84,8 @@ public DefaultSecretSettings getSecretSettings() {
         return (DefaultSecretSettings) super.getSecretSettings();
     }
 
-    public static URI buildUri(String model) throws URISyntaxException {
+    // visible for testing
+    static URI buildUri(String model) throws URISyntaxException {
         return new URIBuilder().setScheme("https")
             .setHost(GoogleAiStudioUtils.HOST_SUFFIX)
             .setPathSegments(
@@ -118,9 +95,4 @@ public static URI buildUri(String model) throws URISyntaxException {
             )
             .build();
     }
-
-    @Override
-    public ExecutableAction accept(GoogleAiStudioActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
-        return visitor.create(this, taskSettings);
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java
index 7c0b812ee213b..676fec13d1ddc 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java
@@ -102,7 +102,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
index 5d46a8e129dff..a9434fb473599 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
@@ -11,13 +11,10 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.EmptyTaskSettings;
-import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.TaskSettings;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
-import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel;
@@ -139,11 +136,6 @@ public URI uri() {
         return uri;
     }
 
-    @Override
-    public ExecutableAction accept(GoogleAiStudioActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
-        return visitor.create(this, taskSettings);
-    }
-
     public static URI buildUri(String model) throws URISyntaxException {
         return new URIBuilder().setScheme("https")
             .setHost(GoogleAiStudioUtils.HOST_SUFFIX)
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
index 7608f48d0638d..2ddf8efcbc112 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
@@ -154,7 +154,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java
index 57c8d61f9f9a5..44e16fa058506 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -30,7 +31,7 @@ public class GoogleVertexAiSecretSettings implements SecretSettings {
 
     public static final String SERVICE_ACCOUNT_JSON = "service_account_json";
 
-    private final SecureString serviceAccountJson;
+    final SecureString serviceAccountJson;
 
     public static GoogleVertexAiSecretSettings fromMap(@Nullable Map<String, Object> map) {
         if (map == null) {
@@ -81,7 +82,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_VERTEX_AI_EMBEDDINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
@@ -101,4 +102,9 @@ public boolean equals(Object object) {
     public int hashCode() {
         return Objects.hash(serviceAccountJson);
     }
+
+    @Override
+    public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+        return GoogleVertexAiSecretSettings.fromMap(new HashMap<>(newSecrets));
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java
index ece23d4226fcd..36fb183f6de70 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java
@@ -151,7 +151,7 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_VERTEX_AI_EMBEDDINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java
index 097ce6240439b..32c3008a802c2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java
@@ -215,7 +215,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_VERTEX_AI_EMBEDDINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java
index 5e0185a7abb36..dcdbbda33575f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -77,7 +78,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_VERTEX_AI_EMBEDDINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
@@ -107,4 +108,12 @@ public boolean equals(Object object) {
     public int hashCode() {
         return Objects.hash(autoTruncate);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        GoogleVertexAiEmbeddingsRequestTaskSettings requestSettings = GoogleVertexAiEmbeddingsRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, requestSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java
index 431f704c10091..d123d14b67f0e 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java
@@ -99,7 +99,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_VERTEX_AI_RERANKING_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java
index 8256eed7a5cba..0bc3df581d7b2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -74,7 +75,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_VERTEX_AI_RERANKING_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
@@ -107,4 +108,12 @@ public boolean equals(Object object) {
     public int hashCode() {
         return Objects.hash(topN);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        GoogleVertexAiRerankRequestTaskSettings requestSettings = GoogleVertexAiRerankRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, requestSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
index 14af29bab9078..752d1dd605cd7 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
@@ -144,6 +144,6 @@ public String name() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java
index c136e73a0c452..7429153835ee3 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java
@@ -120,7 +120,7 @@ public HuggingFaceServiceSettings(StreamInput in) throws IOException {
             maxInputTokens = null;
         }
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings = new RateLimitSettings(in);
         } else {
             rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS;
@@ -171,7 +171,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalVInt(maxInputTokens);
         }
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java
index b48e71867ea6d..be5f595d2c0fb 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java
@@ -76,7 +76,7 @@ public HuggingFaceElserServiceSettings(String url) {
     public HuggingFaceElserServiceSettings(StreamInput in) throws IOException {
         uri = createUri(in.readString());
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings = new RateLimitSettings(in);
         } else {
             rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS;
@@ -134,7 +134,7 @@ public TransportVersion getMinimalSupportedVersion() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(uri.toString());
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
index 8af817d06fefa..8ae9b91b599d9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
@@ -41,7 +42,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.TransportVersions.ADD_MISTRAL_EMBEDDINGS_INFERENCE;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
@@ -206,7 +206,7 @@ public Model parsePersistedConfig(String modelId, TaskType taskType, Map<String,
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return ADD_MISTRAL_EMBEDDINGS_INFERENCE;
+        return TransportVersions.V_8_15_0;
     }
 
     private static MistralEmbeddingsModel createModel(
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
index 3310b8624e723..6b1c7d36a9fe6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.services.mistral.embeddings;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -26,7 +27,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.TransportVersions.ADD_MISTRAL_EMBEDDINGS_INFERENCE;
 import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS;
 import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS;
 import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY;
@@ -104,7 +104,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return ADD_MISTRAL_EMBEDDINGS_INFERENCE;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
index 3161493d68d0c..30656d004b1d2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
@@ -312,7 +312,7 @@ public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
index 96436996098f8..88840cc1202ac 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
@@ -119,7 +119,7 @@ public OpenAiChatCompletionServiceSettings(StreamInput in) throws IOException {
         this.organizationId = in.readOptionalString();
         this.maxInputTokens = in.readOptionalVInt();
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings = new RateLimitSettings(in);
         } else {
             rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS;
@@ -197,7 +197,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(organizationId);
         out.writeOptionalVInt(maxInputTokens);
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java
index 3c2586fb5a264..44064f61f5180 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -107,4 +108,12 @@ public boolean equals(Object object) {
     public int hashCode() {
         return Objects.hash(user);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        OpenAiChatCompletionRequestTaskSettings updatedSettings = OpenAiChatCompletionRequestTaskSettings.fromMap(
+            new HashMap<>(newSettings)
+        );
+        return of(this, updatedSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
index 940f78bcd2ca1..4bb87f41be2e6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
@@ -202,7 +202,7 @@ public OpenAiEmbeddingsServiceSettings(StreamInput in) throws IOException {
             modelId = "unset";
         }
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings = new RateLimitSettings(in);
         } else {
             rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS;
@@ -330,7 +330,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(modelId);
         }
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             rateLimitSettings.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java
index c7cc60043ef47..64f852822703c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -127,4 +128,10 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(user);
     }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        OpenAiEmbeddingsRequestTaskSettings requestSettings = OpenAiEmbeddingsRequestTaskSettings.fromMap(new HashMap<>(newSettings));
+        return of(this, requestSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java
index 6affa998c089d..c68d4bc801724 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -78,4 +79,9 @@ public TransportVersion getMinimalSupportedVersion() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeSecureString(apiKey);
     }
+
+    @Override
+    public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+        return fromMap(new HashMap<>(newSecrets));
+    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java
index b50ea9e5ee224..d27a326d5fa1e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference;
 
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.inference.EmptySecretSettings;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
@@ -32,4 +33,13 @@ protected EmptySecretSettings mutateInstance(EmptySecretSettings instance) {
         // All instances are the same and have no fields, nothing to mutate
         return null;
     }
+
+    public void testNewSecretSettings() {
+
+        EmptySecretSettings newSecretSettings = (EmptySecretSettings) EmptySecretSettings.INSTANCE.newSecretSettings(
+            randomMap(0, 3, () -> new Tuple<>(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))
+        );
+
+        assertSame(EmptySecretSettings.INSTANCE, newSecretSettings);
+    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java
index 060dc23b935cc..7bc0cc57e31ab 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java
@@ -11,12 +11,20 @@
 import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Map;
+
 public class EmptyTaskSettingsTests extends AbstractWireSerializingTestCase<EmptyTaskSettings> {
 
     public static EmptyTaskSettings createRandom() {
         return EmptyTaskSettings.INSTANCE; // no options to randomise
     }
 
+    public void testUpdatedTaskSettings() {
+        EmptyTaskSettings initialSettings = createRandom();
+        EmptyTaskSettings updatedSettings = (EmptyTaskSettings) initialSettings.updatedTaskSettings(Map.of());
+        assertEquals(EmptyTaskSettings.INSTANCE, updatedSettings);
+    }
+
     @Override
     protected Writeable.Reader<EmptyTaskSettings> instanceReader() {
         return EmptyTaskSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java
index d6d139190c12c..ea2f41bf5c6cf 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 public class ModelSecretsTests extends AbstractWireSerializingTestCase<ModelSecrets> {
 
@@ -83,5 +84,10 @@ public String getWriteableName() {
         public TransportVersion getMinimalSupportedVersion() {
             return TransportVersions.V_8_11_X;
         }
+
+        @Override
+        public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+            return new FakeSecretSettings(newSecrets.get(API_KEY).toString());
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/GetInferenceModelRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/GetInferenceModelRequestTests.java
index 93694f167259f..314b3037fdd63 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/GetInferenceModelRequestTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/GetInferenceModelRequestTests.java
@@ -15,7 +15,7 @@
 public class GetInferenceModelRequestTests extends AbstractWireSerializingTestCase<GetInferenceModelAction.Request> {
 
     public static GetInferenceModelAction.Request randomTestInstance() {
-        return new GetInferenceModelAction.Request(randomAlphaOfLength(8), randomFrom(TaskType.values()));
+        return new GetInferenceModelAction.Request(randomAlphaOfLength(8), randomFrom(TaskType.values()), randomBoolean());
     }
 
     @Override
@@ -30,12 +30,17 @@ protected GetInferenceModelAction.Request createTestInstance() {
 
     @Override
     protected GetInferenceModelAction.Request mutateInstance(GetInferenceModelAction.Request instance) {
-        return switch (randomIntBetween(0, 1)) {
+        return switch (randomIntBetween(0, 2)) {
             case 0 -> new GetInferenceModelAction.Request(instance.getInferenceEntityId() + "foo", instance.getTaskType());
             case 1 -> {
                 var nextTaskType = TaskType.values()[(instance.getTaskType().ordinal() + 1) % TaskType.values().length];
                 yield new GetInferenceModelAction.Request(instance.getInferenceEntityId(), nextTaskType);
             }
+            case 2 -> new GetInferenceModelAction.Request(
+                instance.getInferenceEntityId(),
+                instance.getTaskType(),
+                instance.isPersistDefaultConfig() == false
+            );
             default -> throw new UnsupportedOperationException();
         };
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java
index 27e56c1bd973d..991c5a581eb35 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -17,27 +18,18 @@ public class TransportPutInferenceModelActionTests extends ESTestCase {
 
     public void testResolveTaskType() {
 
-        assertEquals(TaskType.SPARSE_EMBEDDING, TransportPutInferenceModelAction.resolveTaskType(TaskType.SPARSE_EMBEDDING, null));
-        assertEquals(
-            TaskType.SPARSE_EMBEDDING,
-            TransportPutInferenceModelAction.resolveTaskType(TaskType.ANY, TaskType.SPARSE_EMBEDDING.toString())
-        );
+        assertEquals(TaskType.SPARSE_EMBEDDING, ServiceUtils.resolveTaskType(TaskType.SPARSE_EMBEDDING, null));
+        assertEquals(TaskType.SPARSE_EMBEDDING, ServiceUtils.resolveTaskType(TaskType.ANY, TaskType.SPARSE_EMBEDDING.toString()));
 
-        var e = expectThrows(
-            ElasticsearchStatusException.class,
-            () -> TransportPutInferenceModelAction.resolveTaskType(TaskType.ANY, null)
-        );
+        var e = expectThrows(ElasticsearchStatusException.class, () -> ServiceUtils.resolveTaskType(TaskType.ANY, null));
         assertThat(e.getMessage(), containsString("model is missing required setting [task_type]"));
 
-        e = expectThrows(
-            ElasticsearchStatusException.class,
-            () -> TransportPutInferenceModelAction.resolveTaskType(TaskType.ANY, TaskType.ANY.toString())
-        );
+        e = expectThrows(ElasticsearchStatusException.class, () -> ServiceUtils.resolveTaskType(TaskType.ANY, TaskType.ANY.toString()));
         assertThat(e.getMessage(), containsString("task_type [any] is not valid type for inference"));
 
         e = expectThrows(
             ElasticsearchStatusException.class,
-            () -> TransportPutInferenceModelAction.resolveTaskType(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING.toString())
+            () -> ServiceUtils.resolveTaskType(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING.toString())
         );
         assertThat(
             e.getMessage(),
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java
index 3c09984ac0162..235a3730ce4f6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java
@@ -17,7 +17,7 @@
 
 public class ChunkingSettingsBuilderTests extends ESTestCase {
 
-    public static final WordBoundaryChunkingSettings DEFAULT_SETTINGS = new WordBoundaryChunkingSettings(250, 100);
+    public static final SentenceBoundaryChunkingSettings DEFAULT_SETTINGS = new SentenceBoundaryChunkingSettings(250, 1);
 
     public void testEmptyChunkingSettingsMap() {
         ChunkingSettings chunkingSettings = ChunkingSettingsBuilder.fromMap(Collections.emptyMap());
@@ -38,25 +38,27 @@ public void testValidChunkingSettingsMap() {
     }
 
     private Map<Map<String, Object>, ChunkingSettings> chunkingSettingsMapToChunkingSettings() {
-        var maxChunkSize = randomNonNegativeInt();
-        var overlap = randomIntBetween(1, maxChunkSize / 2);
+        var maxChunkSizeWordBoundaryChunkingSettings = randomIntBetween(10, 300);
+        var overlap = randomIntBetween(1, maxChunkSizeWordBoundaryChunkingSettings / 2);
+        var maxChunkSizeSentenceBoundaryChunkingSettings = randomIntBetween(20, 300);
+
         return Map.of(
             Map.of(
                 ChunkingSettingsOptions.STRATEGY.toString(),
                 ChunkingStrategy.WORD.toString(),
                 ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(),
-                maxChunkSize,
+                maxChunkSizeWordBoundaryChunkingSettings,
                 ChunkingSettingsOptions.OVERLAP.toString(),
                 overlap
             ),
-            new WordBoundaryChunkingSettings(maxChunkSize, overlap),
+            new WordBoundaryChunkingSettings(maxChunkSizeWordBoundaryChunkingSettings, overlap),
             Map.of(
                 ChunkingSettingsOptions.STRATEGY.toString(),
                 ChunkingStrategy.SENTENCE.toString(),
                 ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(),
-                maxChunkSize
+                maxChunkSizeSentenceBoundaryChunkingSettings
             ),
-            new SentenceBoundaryChunkingSettings(maxChunkSize, 1)
+            new SentenceBoundaryChunkingSettings(maxChunkSizeSentenceBoundaryChunkingSettings, 1)
         );
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java
index 8373ae93354b1..2832c2f64e0e6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java
@@ -21,11 +21,11 @@ public static ChunkingSettings createRandomChunkingSettings() {
 
         switch (randomStrategy) {
             case WORD -> {
-                var maxChunkSize = randomNonNegativeInt();
+                var maxChunkSize = randomIntBetween(10, 300);
                 return new WordBoundaryChunkingSettings(maxChunkSize, randomIntBetween(1, maxChunkSize / 2));
             }
             case SENTENCE -> {
-                return new SentenceBoundaryChunkingSettings(randomNonNegativeInt(), randomBoolean() ? 0 : 1);
+                return new SentenceBoundaryChunkingSettings(randomIntBetween(20, 300), randomBoolean() ? 0 : 1);
             }
             default -> throw new IllegalArgumentException("Unsupported random strategy [" + randomStrategy + "]");
         }
@@ -38,13 +38,13 @@ public static Map<String, Object> createRandomChunkingSettingsMap() {
 
         switch (randomStrategy) {
             case WORD -> {
-                var maxChunkSize = randomNonNegativeInt();
+                var maxChunkSize = randomIntBetween(10, 300);
                 chunkingSettingsMap.put(ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), maxChunkSize);
                 chunkingSettingsMap.put(ChunkingSettingsOptions.OVERLAP.toString(), randomIntBetween(1, maxChunkSize / 2));
 
             }
             case SENTENCE -> {
-                chunkingSettingsMap.put(ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), randomNonNegativeInt());
+                chunkingSettingsMap.put(ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), randomIntBetween(20, 300));
             }
             default -> {
             }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java
index 5687ebc4dbae7..afce8c57e0350 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java
@@ -318,7 +318,8 @@ public void testChunkSplitLargeChunkSizesWithChunkingSettings() {
     }
 
     public void testInvalidChunkingSettingsProvided() {
-        ChunkingSettings chunkingSettings = new WordBoundaryChunkingSettings(randomNonNegativeInt(), randomNonNegativeInt());
+        var maxChunkSize = randomIntBetween(10, 300);
+        ChunkingSettings chunkingSettings = new WordBoundaryChunkingSettings(maxChunkSize, randomIntBetween(1, maxChunkSize / 2));
         assertThrows(IllegalArgumentException.class, () -> { new SentenceBoundaryChunker().chunk(TEST_TEXT, chunkingSettings); });
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java
index fe97d7eb3af54..47a1a116ba21e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.inference.ChunkingStrategy;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -28,14 +27,14 @@ public void testMaxChunkSizeNotProvided() {
     }
 
     public void testInvalidInputsProvided() {
-        var chunkingSettingsMap = buildChunkingSettingsMap(Optional.of(randomNonNegativeInt()));
+        var chunkingSettingsMap = buildChunkingSettingsMap(Optional.of(randomIntBetween(20, 300)));
         chunkingSettingsMap.put(randomAlphaOfLength(10), randomNonNegativeInt());
 
         assertThrows(ValidationException.class, () -> { SentenceBoundaryChunkingSettings.fromMap(chunkingSettingsMap); });
     }
 
     public void testValidInputsProvided() {
-        int maxChunkSize = randomNonNegativeInt();
+        int maxChunkSize = randomIntBetween(20, 300);
         SentenceBoundaryChunkingSettings settings = SentenceBoundaryChunkingSettings.fromMap(
             buildChunkingSettingsMap(Optional.of(maxChunkSize))
         );
@@ -59,12 +58,12 @@ protected Writeable.Reader<SentenceBoundaryChunkingSettings> instanceReader() {
 
     @Override
     protected SentenceBoundaryChunkingSettings createTestInstance() {
-        return new SentenceBoundaryChunkingSettings(randomNonNegativeInt(), randomBoolean() ? 0 : 1);
+        return new SentenceBoundaryChunkingSettings(randomIntBetween(20, 300), randomBoolean() ? 0 : 1);
     }
 
     @Override
     protected SentenceBoundaryChunkingSettings mutateInstance(SentenceBoundaryChunkingSettings instance) throws IOException {
-        var chunkSize = randomValueOtherThan(instance.maxChunkSize, ESTestCase::randomNonNegativeInt);
+        var chunkSize = randomValueOtherThan(instance.maxChunkSize, () -> randomIntBetween(20, 300));
         return new SentenceBoundaryChunkingSettings(chunkSize, instance.sentenceOverlap);
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java
index 08c0724f36270..ef643a4b36fdc 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java
@@ -136,7 +136,7 @@ public void testNumberOfChunksWithWordBoundaryChunkingSettings() {
     }
 
     public void testInvalidChunkingSettingsProvided() {
-        ChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(randomNonNegativeInt(), 0);
+        ChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(randomIntBetween(20, 300), 0);
         assertThrows(IllegalArgumentException.class, () -> { new WordBoundaryChunker().chunk(TEST_TEXT, chunkingSettings); });
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettingsTests.java
index c5515f7bf0512..dd91a3c7a947e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettingsTests.java
@@ -14,7 +14,6 @@
 
 import java.io.IOException;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
@@ -28,19 +27,20 @@ public void testMaxChunkSizeNotProvided() {
 
     public void testOverlapNotProvided() {
         assertThrows(ValidationException.class, () -> {
-            WordBoundaryChunkingSettings.fromMap(buildChunkingSettingsMap(Optional.of(randomNonNegativeInt()), Optional.empty()));
+            WordBoundaryChunkingSettings.fromMap(buildChunkingSettingsMap(Optional.of(randomIntBetween(10, 300)), Optional.empty()));
         });
     }
 
     public void testInvalidInputsProvided() {
-        var chunkingSettingsMap = buildChunkingSettingsMap(Optional.of(randomNonNegativeInt()), Optional.of(randomNonNegativeInt()));
+        var maxChunkSize = randomIntBetween(10, 300);
+        var chunkingSettingsMap = buildChunkingSettingsMap(Optional.of(maxChunkSize), Optional.of(randomIntBetween(1, maxChunkSize / 2)));
         chunkingSettingsMap.put(randomAlphaOfLength(10), randomNonNegativeInt());
 
         assertThrows(ValidationException.class, () -> { WordBoundaryChunkingSettings.fromMap(chunkingSettingsMap); });
     }
 
     public void testOverlapGreaterThanHalfMaxChunkSize() {
-        var maxChunkSize = randomNonNegativeInt();
+        var maxChunkSize = randomIntBetween(10, 300);
         var overlap = randomIntBetween((maxChunkSize / 2) + 1, maxChunkSize);
         assertThrows(ValidationException.class, () -> {
             WordBoundaryChunkingSettings.fromMap(buildChunkingSettingsMap(Optional.of(maxChunkSize), Optional.of(overlap)));
@@ -48,7 +48,7 @@ public void testOverlapGreaterThanHalfMaxChunkSize() {
     }
 
     public void testValidInputsProvided() {
-        int maxChunkSize = randomNonNegativeInt();
+        int maxChunkSize = randomIntBetween(10, 300);
         int overlap = randomIntBetween(1, maxChunkSize / 2);
         WordBoundaryChunkingSettings settings = WordBoundaryChunkingSettings.fromMap(
             buildChunkingSettingsMap(Optional.of(maxChunkSize), Optional.of(overlap))
@@ -75,29 +75,14 @@ protected Writeable.Reader<WordBoundaryChunkingSettings> instanceReader() {
 
     @Override
     protected WordBoundaryChunkingSettings createTestInstance() {
-        var maxChunkSize = randomNonNegativeInt();
+        var maxChunkSize = randomIntBetween(10, 300);
         return new WordBoundaryChunkingSettings(maxChunkSize, randomIntBetween(1, maxChunkSize / 2));
     }
 
     @Override
     protected WordBoundaryChunkingSettings mutateInstance(WordBoundaryChunkingSettings instance) throws IOException {
-        var valueToMutate = randomFrom(List.of(ChunkingSettingsOptions.MAX_CHUNK_SIZE, ChunkingSettingsOptions.OVERLAP));
-        var maxChunkSize = instance.maxChunkSize;
-        var overlap = instance.overlap;
-
-        if (valueToMutate.equals(ChunkingSettingsOptions.MAX_CHUNK_SIZE)) {
-            while (maxChunkSize == instance.maxChunkSize) {
-                maxChunkSize = randomNonNegativeInt();
-            }
-
-            if (overlap > maxChunkSize / 2) {
-                overlap = randomIntBetween(1, maxChunkSize / 2);
-            }
-        } else if (valueToMutate.equals(ChunkingSettingsOptions.OVERLAP)) {
-            while (overlap == instance.overlap) {
-                overlap = randomIntBetween(1, maxChunkSize / 2);
-            }
-        }
+        var maxChunkSize = randomValueOtherThan(instance.maxChunkSize, () -> randomIntBetween(10, 300));
+        var overlap = randomValueOtherThan(instance.overlap, () -> randomIntBetween(1, maxChunkSize / 2));
 
         return new WordBoundaryChunkingSettings(maxChunkSize, overlap);
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java
index 6fcd386702497..72b5ffa45a0dd 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java
@@ -272,7 +272,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc
     private ExecutableAction createAction(String url, String apiKey, String modelName, Sender sender) {
         var model = GoogleAiStudioCompletionModelTests.createModel(modelName, url, apiKey);
         var requestManager = new GoogleAiStudioCompletionRequestManager(model, threadPool);
-        var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio completion");
+        var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(false), "Google AI Studio completion");
         return new SingleInputSenderExecutableAction(
             sender,
             requestManager,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java
index 8f09c53c99366..6d601b4b08c53 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockChatCompletionRequest;
-import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockTitanCompletionRequestEntity;
+import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestEntity;
 import org.elasticsearch.xpack.inference.external.request.amazonbedrock.embeddings.AmazonBedrockEmbeddingsRequest;
 import org.elasticsearch.xpack.inference.external.request.amazonbedrock.embeddings.AmazonBedrockTitanEmbeddingsRequestEntity;
 import org.elasticsearch.xpack.inference.external.response.amazonbedrock.completion.AmazonBedrockChatCompletionResponseHandler;
@@ -100,8 +100,8 @@ public void testExecute_ChatCompletionRequest() throws CharacterCodingException
             "secretkey"
         );
 
-        var requestEntity = new AmazonBedrockTitanCompletionRequestEntity(List.of("abc"), null, null, 512);
-        var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null);
+        var requestEntity = new AmazonBedrockConverseRequestEntity(List.of("abc"), null, null, 512);
+        var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null, false);
         var responseHandler = new AmazonBedrockChatCompletionResponseHandler();
 
         var clientCache = new AmazonBedrockMockClientCache(getTestConverseResult("converse result"), null, null);
@@ -124,8 +124,8 @@ public void testExecute_FailsProperly_WithElasticsearchException() {
             "secretkey"
         );
 
-        var requestEntity = new AmazonBedrockTitanCompletionRequestEntity(List.of("abc"), null, null, 512);
-        var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null);
+        var requestEntity = new AmazonBedrockConverseRequestEntity(List.of("abc"), null, null, 512);
+        var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null, false);
         var responseHandler = new AmazonBedrockChatCompletionResponseHandler();
 
         var clientCache = new AmazonBedrockMockClientCache(null, null, new ElasticsearchException("test exception"));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java
index 873b2e22497c6..bb7c669cdf09b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java
@@ -25,7 +25,7 @@
 public class AmazonBedrockInferenceClientCacheTests extends ESTestCase {
     public void testCache_ReturnsSameObject() throws IOException {
         AmazonBedrockInferenceClientCache cacheInstance;
-        try (var cache = new AmazonBedrockInferenceClientCache(AmazonBedrockMockInferenceClient::create, null)) {
+        try (var cache = new AmazonBedrockInferenceClientCache(AmazonBedrockMockInferenceClient::create, Clock.systemUTC())) {
             cacheInstance = cache;
             var model = AmazonBedrockEmbeddingsModelTests.createModel(
                 "inferenceId",
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java
index 5584e90b3264d..e6cd667b824b3 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java
@@ -14,15 +14,19 @@
 import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockModel;
 
 import java.util.concurrent.CompletableFuture;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class AmazonBedrockMockInferenceClient extends AmazonBedrockInferenceClient {
     private CompletableFuture<ConverseResponse> converseResponseFuture = CompletableFuture.completedFuture(null);
@@ -33,7 +37,13 @@ public static AmazonBedrockMockInferenceClient create(AmazonBedrockModel model,
     }
 
     protected AmazonBedrockMockInferenceClient(AmazonBedrockModel model, @Nullable TimeValue timeout) {
-        super(model, timeout);
+        super(model, timeout, mockThreadPool());
+    }
+
+    private static ThreadPool mockThreadPool() {
+        ThreadPool threadPool = mock();
+        when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
+        return threadPool;
     }
 
     public void setExceptionToThrow(ElasticsearchException exceptionToThrow) {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessorTests.java
new file mode 100644
index 0000000000000..ba87bdfe16cdd
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessorTests.java
@@ -0,0 +1,251 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.amazonbedrock;
+
+import software.amazon.awssdk.services.bedrockruntime.model.BedrockRuntimeException;
+import software.amazon.awssdk.services.bedrockruntime.model.ContentBlockDelta;
+import software.amazon.awssdk.services.bedrockruntime.model.ContentBlockDeltaEvent;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamOutput;
+import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Flow;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isA;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.assertArg;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class AmazonBedrockStreamingChatProcessorTests extends ESTestCase {
+    private AmazonBedrockStreamingChatProcessor processor;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        ThreadPool threadPool = mock();
+        when(threadPool.executor(UTILITY_THREAD_POOL_NAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
+        processor = new AmazonBedrockStreamingChatProcessor(threadPool);
+    }
+
+    /**
+     * We do not issue requests on subscribe because the downstream will control the pacing.
+     */
+    public void testOnSubscribeBeforeDownstreamDoesNotRequest() {
+        var upstream = mock(Flow.Subscription.class);
+        processor.onSubscribe(upstream);
+
+        verify(upstream, never()).request(anyLong());
+    }
+
+    /**
+     * If the downstream requests data before the upstream is set, when the upstream is set, we will forward the pending requests to it.
+     */
+    public void testOnSubscribeAfterDownstreamRequests() {
+        var expectedRequestCount = randomLongBetween(1, 500);
+        Flow.Subscriber<StreamingChatCompletionResults.Results> subscriber = mock();
+        doAnswer(ans -> {
+            Flow.Subscription sub = ans.getArgument(0);
+            sub.request(expectedRequestCount);
+            return null;
+        }).when(subscriber).onSubscribe(any());
+        processor.subscribe(subscriber);
+
+        var upstream = mock(Flow.Subscription.class);
+        processor.onSubscribe(upstream);
+
+        verify(upstream, times(1)).request(anyLong());
+    }
+
+    public void testCancelDuplicateSubscriptions() {
+        processor.onSubscribe(mock());
+
+        var upstream = mock(Flow.Subscription.class);
+        processor.onSubscribe(upstream);
+
+        verify(upstream, times(1)).cancel();
+        verifyNoMoreInteractions(upstream);
+    }
+
+    public void testMultiplePublishesCallsOnError() {
+        processor.subscribe(mock());
+
+        Flow.Subscriber<StreamingChatCompletionResults.Results> subscriber = mock();
+        processor.subscribe(subscriber);
+
+        verify(subscriber, times(1)).onError(assertArg(e -> {
+            assertThat(e, isA(IllegalStateException.class));
+            assertThat(e.getMessage(), equalTo("Subscriber already set."));
+        }));
+    }
+
+    public void testNonDeltaBlocksAreSkipped() {
+        var upstream = mock(Flow.Subscription.class);
+        processor.onSubscribe(upstream);
+        var counter = new AtomicInteger();
+        Arrays.stream(ConverseStreamOutput.EventType.values())
+            .filter(type -> type != ConverseStreamOutput.EventType.CONTENT_BLOCK_DELTA)
+            .forEach(type -> {
+                ConverseStreamOutput output = mock();
+                when(output.sdkEventType()).thenReturn(type);
+                processor.onNext(output);
+                verify(upstream, times(counter.incrementAndGet())).request(eq(1L));
+            });
+    }
+
+    public void testDeltaBlockForwardsDownstream() {
+        var expectedText = "hello";
+
+        // mock executorservice so we can make sure we handle the response on another thread
+        ExecutorService executorService = mock();
+        ThreadPool threadPool = mock();
+        when(threadPool.executor(UTILITY_THREAD_POOL_NAME)).thenReturn(executorService);
+        processor = new AmazonBedrockStreamingChatProcessor(threadPool);
+        doAnswer(ans -> {
+            Runnable command = ans.getArgument(0);
+            command.run();
+            return null;
+        }).when(executorService).execute(any());
+
+        Flow.Subscription upstream = mock();
+        processor.onSubscribe(upstream);
+        Flow.Subscriber<StreamingChatCompletionResults.Results> downstream = mock();
+        processor.subscribe(downstream);
+
+        ConverseStreamOutput output = output(expectedText);
+
+        processor.onNext(output);
+
+        verifyText(downstream, expectedText);
+        verify(executorService, times(1)).execute(any());
+        verify(upstream, times(0)).request(anyLong());
+    }
+
+    private ConverseStreamOutput output(String text) {
+        ConverseStreamOutput output = mock();
+        when(output.sdkEventType()).thenReturn(ConverseStreamOutput.EventType.CONTENT_BLOCK_DELTA);
+        doAnswer(ans -> {
+            ConverseStreamResponseHandler.Visitor visitor = ans.getArgument(0);
+            ContentBlockDelta delta = ContentBlockDelta.fromText(text);
+            ContentBlockDeltaEvent event = ContentBlockDeltaEvent.builder().delta(delta).build();
+            visitor.visitContentBlockDelta(event);
+            return null;
+        }).when(output).accept(any());
+        return output;
+    }
+
+    private void verifyText(Flow.Subscriber<StreamingChatCompletionResults.Results> downstream, String expectedText) {
+        verify(downstream, times(1)).onNext(assertArg(results -> {
+            assertThat(results, notNullValue());
+            assertThat(results.results().size(), equalTo(1));
+            assertThat(results.results().getFirst().delta(), equalTo(expectedText));
+        }));
+    }
+
+    public void verifyCompleteBeforeRequest() {
+        processor.onComplete();
+
+        Flow.Subscriber<StreamingChatCompletionResults.Results> downstream = mock();
+        var sub = ArgumentCaptor.forClass(Flow.Subscription.class);
+        processor.subscribe(downstream);
+        verify(downstream).onSubscribe(sub.capture());
+
+        sub.getValue().request(1);
+        verify(downstream, times(1)).onComplete();
+    }
+
+    public void verifyCompleteAfterRequest() {
+
+        Flow.Subscriber<StreamingChatCompletionResults.Results> downstream = mock();
+        var sub = ArgumentCaptor.forClass(Flow.Subscription.class);
+        processor.subscribe(downstream);
+        verify(downstream).onSubscribe(sub.capture());
+
+        sub.getValue().request(1);
+        processor.onComplete();
+        verify(downstream, times(1)).onComplete();
+    }
+
+    public void verifyOnErrorBeforeRequest() {
+        var expectedError = BedrockRuntimeException.builder().message("ahhhhhh").build();
+        processor.onError(expectedError);
+
+        Flow.Subscriber<StreamingChatCompletionResults.Results> downstream = mock();
+        var sub = ArgumentCaptor.forClass(Flow.Subscription.class);
+        processor.subscribe(downstream);
+        verify(downstream).onSubscribe(sub.capture());
+
+        sub.getValue().request(1);
+        verify(downstream, times(1)).onError(assertArg(e -> {
+            assertThat(e, isA(ElasticsearchException.class));
+            assertThat(e.getCause(), is(expectedError));
+        }));
+    }
+
+    public void verifyOnErrorAfterRequest() {
+        var expectedError = BedrockRuntimeException.builder().message("ahhhhhh").build();
+
+        Flow.Subscriber<StreamingChatCompletionResults.Results> downstream = mock();
+        var sub = ArgumentCaptor.forClass(Flow.Subscription.class);
+        processor.subscribe(downstream);
+        verify(downstream).onSubscribe(sub.capture());
+
+        sub.getValue().request(1);
+        processor.onError(expectedError);
+        verify(downstream, times(1)).onError(assertArg(e -> {
+            assertThat(e, isA(ElasticsearchException.class));
+            assertThat(e.getCause(), is(expectedError));
+        }));
+    }
+
+    public void verifyAsyncOnCompleteIsStillDeliveredSynchronously() {
+        mockUpstream();
+
+        Flow.Subscriber<StreamingChatCompletionResults.Results> downstream = mock();
+        var sub = ArgumentCaptor.forClass(Flow.Subscription.class);
+        processor.subscribe(downstream);
+        verify(downstream).onSubscribe(sub.capture());
+
+        sub.getValue().request(1);
+        verify(downstream, times(1)).onNext(any());
+        processor.onComplete();
+        verify(downstream, times(0)).onComplete();
+        sub.getValue().request(1);
+        verify(downstream, times(1)).onComplete();
+    }
+
+    private void mockUpstream() {
+        Flow.Subscription upstream = mock();
+        doAnswer(ans -> {
+            processor.onNext(output(randomIdentifier()));
+            return null;
+        }).when(upstream).request(anyLong());
+        processor.onSubscribe(upstream);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java
index 1667dac84d2db..ba6bcf8b57d5b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java
@@ -11,20 +11,17 @@
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
 import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent;
-import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField;
-import org.hamcrest.Matcher;
-import org.hamcrest.Matchers;
 
 import java.util.ArrayDeque;
-import java.util.Arrays;
 import java.util.Deque;
 import java.util.Map;
 import java.util.concurrent.Flow;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onNext;
+import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.containsResults;
+import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.events;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.isA;
 import static org.hamcrest.Matchers.notNullValue;
@@ -133,21 +130,6 @@ public void testDroppedEventsRequestsMoreData() throws Exception {
         verify(downstream, times(0)).onNext(any());
     }
 
-    private Deque<ServerSentEvent> events(String... data) {
-        var item = new ArrayDeque<ServerSentEvent>();
-        Arrays.stream(data).map(datum -> new ServerSentEvent(ServerSentEventField.DATA, datum)).forEach(item::offer);
-        return item;
-    }
-
-    @SuppressWarnings("unchecked")
-    private Matcher<Iterable<? extends StreamingChatCompletionResults.Result>> containsResults(String... results) {
-        Matcher<StreamingChatCompletionResults.Result>[] resultMatcher = Arrays.stream(results)
-            .map(StreamingChatCompletionResults.Result::new)
-            .map(Matchers::equalTo)
-            .toArray(Matcher[]::new);
-        return Matchers.contains(resultMatcher);
-    }
-
     private static ElasticsearchStatusException onError(Deque<ServerSentEvent> item) {
         var processor = new AnthropicStreamingProcessor();
         var response = new AtomicReference<Throwable>();
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessorTests.java
new file mode 100644
index 0000000000000..f41fe5b765c8c
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessorTests.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.googleaistudio;
+
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioCompletionResponseEntity;
+import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent;
+
+import java.util.ArrayDeque;
+import java.util.concurrent.Flow;
+
+import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onError;
+import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onNext;
+import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.containsResults;
+import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.events;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class GoogleAiStudioStreamingProcessorTests extends ESTestCase {
+
+    public void testParseSuccess() {
+        var item = events("""
+             {
+              "candidates": [
+                {
+                  "content": {
+                    "parts": [
+                      {
+                        "text": "Hello"
+                      }
+                    ],
+                    "role": "model"
+                  },
+                  "finishReason": "STOP",
+                  "index": 0,
+                  "safetyRatings": [
+                    {
+                      "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                      "probability": "NEGLIGIBLE"
+                    },
+                    {
+                      "category": "HARM_CATEGORY_HATE_SPEECH",
+                      "probability": "NEGLIGIBLE"
+                    },
+                    {
+                      "category": "HARM_CATEGORY_HARASSMENT",
+                      "probability": "NEGLIGIBLE"
+                    },
+                    {
+                      "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                      "probability": "NEGLIGIBLE"
+                    }
+                  ]
+                }
+              ],
+              "usageMetadata": {
+                "promptTokenCount": 1,
+                "candidatesTokenCount": 1,
+                "totalTokenCount": 1
+              }
+            }""", """
+             {
+              "candidates": [
+                {
+                  "content": {
+                    "parts": [
+                      {
+                        "text": ", World"
+                      }
+                    ],
+                    "role": "model"
+                  },
+                  "finishReason": "STOP",
+                  "index": 0,
+                  "safetyRatings": [
+                    {
+                      "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                      "probability": "NEGLIGIBLE"
+                    },
+                    {
+                      "category": "HARM_CATEGORY_HATE_SPEECH",
+                      "probability": "NEGLIGIBLE"
+                    },
+                    {
+                      "category": "HARM_CATEGORY_HARASSMENT",
+                      "probability": "NEGLIGIBLE"
+                    },
+                    {
+                      "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                      "probability": "NEGLIGIBLE"
+                    }
+                  ]
+                }
+              ],
+              "usageMetadata": {
+                "promptTokenCount": 1,
+                "candidatesTokenCount": 1,
+                "totalTokenCount": 1
+              }
+            }""");
+
+        var response = onNext(new GoogleAiStudioStreamingProcessor(GoogleAiStudioCompletionResponseEntity::content), item);
+        assertThat(response.results().size(), equalTo(2));
+        assertThat(response.results(), containsResults("Hello", ", World"));
+    }
+
+    public void testEmptyResultsRequestsMoreData() throws Exception {
+        var emptyDeque = new ArrayDeque<ServerSentEvent>();
+
+        var processor = new GoogleAiStudioStreamingProcessor(noOp -> {
+            fail("This should not be called");
+            return null;
+        });
+
+        Flow.Subscriber<ChunkedToXContent> downstream = mock();
+        processor.subscribe(downstream);
+
+        Flow.Subscription upstream = mock();
+        processor.onSubscribe(upstream);
+
+        processor.next(emptyDeque);
+
+        verify(upstream, times(1)).request(1);
+        verify(downstream, times(0)).onNext(any());
+    }
+
+    public void testOnError() {
+        var expectedException = new RuntimeException("hello");
+
+        var processor = new GoogleAiStudioStreamingProcessor(noOp -> { throw expectedException; });
+
+        assertThat(onError(processor, events("hi")), sameInstance(expectedException));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java
index be47d8806aade..a400b67b3761f 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.Before;
+import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -38,6 +39,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
@@ -59,7 +61,7 @@ public void setUp() throws Exception {
         super.setUp();
         threadPool = mock(ThreadPool.class);
         settings = mock(HttpSettings.class);
-        listener = ActionListener.noop();
+        listener = spy(ActionListener.noop());
 
         when(threadPool.executor(UTILITY_THREAD_POOL_NAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
         when(settings.getMaxResponseSize()).thenReturn(ByteSizeValue.ofBytes(maxBytes));
@@ -235,33 +237,13 @@ public void testTotalBytesDecrement() throws IOException {
     }
 
     /**
-     * Given an error from Apache
-     * When the subscriber requests the next set of data
-     * Then the subscriber receives the error from Apache
+     * When there is an error from Apache before the publisher invokes the listener
+     * Then the publisher will forward the call to the listener's onFailure
      */
     public void testErrorBeforeRequest() {
-        var subscriber = subscribe();
         var exception = new NullPointerException("test");
-
         publisher.failed(exception);
-        assertThat("subscriber receives exception on next request", subscriber.throwable, nullValue());
-
-        subscriber.requestData();
-        assertThat("subscriber receives exception", subscriber.throwable, is(exception));
-    }
-
-    /**
-     * Given the subscriber is waiting for data
-     * When Apache sends an error
-     * Then the subscriber immediately receives the error
-     */
-    public void testErrorAfterRequest() {
-        var subscriber = subscribe();
-        var exception = new NullPointerException("test");
-
-        subscriber.requestData();
-        publisher.failed(exception);
-        assertThat("subscriber receives exception", subscriber.throwable, is(exception));
+        verify(listener).onFailure(exception);
     }
 
     /**
@@ -375,6 +357,76 @@ public void testCancelAfterRequest() {
         assertTrue("onComplete should be called", subscriber.completed);
     }
 
+    /**
+     * When cancel is called
+     * Then we only send onComplete once
+     */
+    public void testCancelIsIdempotent() throws IOException {
+        Flow.Subscriber<HttpResult> subscriber = mock();
+
+        var subscription = ArgumentCaptor.forClass(Flow.Subscription.class);
+        publisher.subscribe(subscriber);
+        verify(subscriber).onSubscribe(subscription.capture());
+
+        publisher.responseReceived(mock());
+        publisher.consumeContent(contentDecoder(message), mock(IOControl.class));
+        subscription.getValue().request(1);
+
+        subscription.getValue().request(1);
+        publisher.cancel();
+        verify(subscriber, times(1)).onComplete();
+        subscription.getValue().request(1);
+        publisher.cancel();
+        verify(subscriber, times(1)).onComplete();
+    }
+
+    /**
+     * When close is called
+     * Then we only send onComplete once
+     */
+    public void testCloseIsIdempotent() throws IOException {
+        Flow.Subscriber<HttpResult> subscriber = mock();
+
+        var subscription = ArgumentCaptor.forClass(Flow.Subscription.class);
+        publisher.subscribe(subscriber);
+        verify(subscriber).onSubscribe(subscription.capture());
+
+        publisher.responseReceived(mock());
+        publisher.consumeContent(contentDecoder(message), mock(IOControl.class));
+        subscription.getValue().request(1);
+
+        subscription.getValue().request(1);
+        publisher.close();
+        verify(subscriber, times(1)).onComplete();
+        subscription.getValue().request(1);
+        publisher.close();
+        verify(subscriber, times(1)).onComplete();
+    }
+
+    /**
+     * When failed is called
+     * Then we only send onError once
+     */
+    public void testFailedIsIdempotent() throws IOException {
+        var expectedException = new IllegalStateException("wow");
+        Flow.Subscriber<HttpResult> subscriber = mock();
+
+        var subscription = ArgumentCaptor.forClass(Flow.Subscription.class);
+        publisher.subscribe(subscriber);
+        verify(subscriber).onSubscribe(subscription.capture());
+
+        publisher.responseReceived(mock());
+        publisher.consumeContent(contentDecoder(message), mock(IOControl.class));
+        subscription.getValue().request(1);
+
+        subscription.getValue().request(1);
+        publisher.failed(expectedException);
+        verify(subscriber, times(1)).onError(eq(expectedException));
+        subscription.getValue().request(1);
+        publisher.failed(expectedException);
+        verify(subscriber, times(1)).onError(eq(expectedException));
+    }
+
     /**
      * Given the queue is being processed
      * When Apache cancels the publisher
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java
index a57e7c1b64c07..90d0e8742f733 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java
@@ -149,6 +149,42 @@ public void testDoneMessageIsIgnored() throws Exception {
         verify(downstream, times(0)).onNext(any());
     }
 
+    public void testInitialLlamaResponseIsIgnored() throws Exception {
+        var item = new ArrayDeque<ServerSentEvent>();
+        item.offer(new ServerSentEvent(ServerSentEventField.DATA, """
+            {
+                "id":"12345",
+                "object":"chat.completion.chunk",
+                "created":123456789,
+                "model":"Llama-2-7b-chat",
+                "system_fingerprint": "123456789",
+                "choices":[
+                    {
+                        "index":0,
+                        "delta":{
+                            "role":"assistant"
+                        },
+                        "logprobs":null,
+                        "finish_reason":null
+                    }
+                ]
+            }
+            """));
+
+        var processor = new OpenAiStreamingProcessor();
+
+        Flow.Subscriber<ChunkedToXContent> downstream = mock();
+        processor.subscribe(downstream);
+
+        Flow.Subscription upstream = mock();
+        processor.onSubscribe(upstream);
+
+        processor.next(item);
+
+        verify(upstream, times(1)).request(1);
+        verify(downstream, times(0)).onNext(any());
+    }
+
     private String toJsonString(ChunkedToXContent chunkedToXContent) throws IOException {
         try (var builder = XContentFactory.jsonBuilder()) {
             chunkedToXContent.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xContent -> {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntityTests.java
deleted file mode 100644
index 10c8943c75f6c..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntityTests.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest;
-import static org.hamcrest.Matchers.is;
-
-public class AmazonBedrockAI21LabsCompletionRequestEntityTests extends ESTestCase {
-    public void testRequestEntity_CreatesProperRequest() {
-        var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTemperature() {
-        var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), 1.0, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopP() {
-        var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), null, 1.0, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithMaxTokens() {
-        var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), null, null, 128);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128));
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntityTests.java
deleted file mode 100644
index e8a3440a37294..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntityTests.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest;
-import static org.hamcrest.Matchers.is;
-
-public class AmazonBedrockAnthropicCompletionRequestEntityTests extends ESTestCase {
-    public void testRequestEntity_CreatesProperRequest() {
-        var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTemperature() {
-        var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), 1.0, null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopP() {
-        var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, 1.0, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithMaxTokens() {
-        var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, null, null, 128);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopK() {
-        var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, null, 1.0, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopKInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactoryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactoryTests.java
new file mode 100644
index 0000000000000..32cd21bc3d45a
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactoryTests.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider;
+import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionModel;
+import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionServiceSettings;
+import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionTaskSettings;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xcontent.XContentParserConfiguration.EMPTY;
+import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.AI21LABS;
+import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.AMAZONTITAN;
+import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.ANTHROPIC;
+import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.COHERE;
+import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.META;
+import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.MISTRAL;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class AmazonBedrockChatCompletionEntityFactoryTests extends ESTestCase {
+    public void testEntitiesWithoutAdditionalMessages() {
+        List.of(AI21LABS, AMAZONTITAN, META).forEach(provider -> {
+            var expectedTemp = randomDoubleBetween(1, 10, true);
+            var expectedTopP = randomDoubleBetween(1, 10, true);
+
+            var expectedMaxToken = randomIntBetween(1, 10);
+            var expectedMessage = List.of(randomIdentifier());
+            var model = model(provider, expectedTemp, expectedTopP, expectedMaxToken);
+
+            var entity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, expectedMessage);
+
+            assertThat(entity, notNullValue());
+            assertThat(entity.temperature(), equalTo(expectedTemp));
+            assertThat(entity.topP(), equalTo(expectedTopP));
+            assertThat(entity.maxTokenCount(), equalTo(expectedMaxToken));
+            assertThat(entity.additionalModelFields(), nullValue());
+            assertThat(entity.messages(), equalTo(expectedMessage));
+        });
+    }
+
+    public void testWithAdditionalMessages() {
+        List.of(ANTHROPIC, COHERE, MISTRAL).forEach(provider -> {
+            var expectedTemp = randomDoubleBetween(1, 10, true);
+            var expectedTopP = randomDoubleBetween(1, 10, true);
+            var expectedMaxToken = randomIntBetween(1, 10);
+            var expectedMessage = List.of(randomIdentifier());
+            var expectedTopK = randomDoubleBetween(1, 10, true);
+            var model = model(provider, expectedTemp, expectedTopP, expectedMaxToken, expectedTopK);
+
+            var entity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, expectedMessage);
+
+            assertThat(entity, notNullValue());
+            assertThat(entity.temperature(), equalTo(expectedTemp));
+            assertThat(entity.topP(), equalTo(expectedTopP));
+            assertThat(entity.maxTokenCount(), equalTo(expectedMaxToken));
+            assertThat(entity.messages(), equalTo(expectedMessage));
+            assertThat(entity.additionalModelFields(), notNullValue());
+            assertThat(entity.additionalModelFields().size(), equalTo(1));
+            try (var parser = XContentFactory.xContent(XContentType.JSON).createParser(EMPTY, entity.additionalModelFields().getFirst())) {
+                var additionalModelFields = parser.map();
+                assertThat((Double) additionalModelFields.get("top_k"), closeTo(expectedTopK, 0.1));
+            } catch (IOException e) {
+                fail(e);
+            }
+        });
+    }
+
+    AmazonBedrockChatCompletionModel model(AmazonBedrockProvider provider, Double temperature, Double topP, Integer maxTokenCount) {
+        return model(provider, temperature, topP, maxTokenCount, null);
+    }
+
+    AmazonBedrockChatCompletionModel model(AmazonBedrockProvider provider, Double temp, Double topP, Integer tokenCount, Double topK) {
+        var serviceSettings = mock(AmazonBedrockChatCompletionServiceSettings.class);
+        when(serviceSettings.provider()).thenReturn(provider);
+
+        var taskSettings = mock(AmazonBedrockChatCompletionTaskSettings.class);
+        when(taskSettings.temperature()).thenReturn(temp);
+        when(taskSettings.topP()).thenReturn(topP);
+        when(taskSettings.maxNewTokens()).thenReturn(tokenCount);
+        when(taskSettings.topK()).thenReturn(topK);
+
+        var model = mock(AmazonBedrockChatCompletionModel.class);
+        when(model.getServiceSettings()).thenReturn(serviceSettings);
+        when(model.getTaskSettings()).thenReturn(taskSettings);
+        return model;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntityTests.java
deleted file mode 100644
index c8e844d000240..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntityTests.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest;
-import static org.hamcrest.Matchers.is;
-
-public class AmazonBedrockCohereCompletionRequestEntityTests extends ESTestCase {
-    public void testRequestEntity_CreatesProperRequest() {
-        var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTemperature() {
-        var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), 1.0, null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopP() {
-        var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, 1.0, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithMaxTokens() {
-        var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, null, null, 128);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopK() {
-        var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, null, 1.0, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopKInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java
index 17c3b4488bae4..0e7acd3337e0f 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java
@@ -15,12 +15,16 @@
 
 import java.util.Collection;
 
+import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList;
+import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.inferenceConfig;
+
 public final class AmazonBedrockConverseRequestUtils {
     public static ConverseRequest getConverseRequest(String modelId, AmazonBedrockConverseRequestEntity requestEntity) {
-        var converseRequest = ConverseRequest.builder().modelId(modelId);
-        converseRequest = requestEntity.addMessages(converseRequest);
-        converseRequest = requestEntity.addInferenceConfig(converseRequest);
-        converseRequest = requestEntity.addAdditionalModelFields(converseRequest);
+        var converseRequest = ConverseRequest.builder()
+            .modelId(modelId)
+            .messages(getConverseMessageList(requestEntity.messages()))
+            .additionalModelResponseFieldPaths(requestEntity.additionalModelFields());
+        inferenceConfig(requestEntity).ifPresent(converseRequest::inferenceConfig);
         return converseRequest.build();
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntityTests.java
deleted file mode 100644
index 25700f7c7aee1..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntityTests.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest;
-import static org.hamcrest.Matchers.is;
-
-public class AmazonBedrockMetaCompletionRequestEntityTests extends ESTestCase {
-    public void testRequestEntity_CreatesProperRequest() {
-        var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTemperature() {
-        var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), 1.0, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopP() {
-        var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), null, 1.0, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithMaxTokens() {
-        var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), null, null, 128);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128));
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntityTests.java
deleted file mode 100644
index 8e321b0cb33a7..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntityTests.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest;
-import static org.hamcrest.Matchers.is;
-
-public class AmazonBedrockMistralCompletionRequestEntityTests extends ESTestCase {
-    public void testRequestEntity_CreatesProperRequest() {
-        var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTemperature() {
-        var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), 1.0, null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopP() {
-        var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, 1.0, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithMaxTokens() {
-        var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, null, null, 128);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopK() {
-        var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, null, 1.0, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopKInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntityTests.java
deleted file mode 100644
index 8d1c15499bfb6..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntityTests.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput;
-import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest;
-import static org.hamcrest.Matchers.is;
-
-public class AmazonBedrockTitanCompletionRequestEntityTests extends ESTestCase {
-    public void testRequestEntity_CreatesProperRequest() {
-        var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), null, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTemperature() {
-        var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), 1.0, null, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithTopP() {
-        var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), null, 1.0, null);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest));
-    }
-
-    public void testRequestEntity_CreatesProperRequest_WithMaxTokens() {
-        var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), null, null, 128);
-        var builtRequest = getConverseRequest("testmodel", request);
-        assertThat(builtRequest.modelId(), is("testmodel"));
-        assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true));
-        assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest));
-        assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest));
-        assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128));
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java
index 7d7ee1dcba6c2..7ffa8940ad6be 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java
@@ -10,6 +10,7 @@
 import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
 import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioCompletionRequest;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModelTests;
 
@@ -29,7 +30,7 @@ public void testCreateRequest() throws IOException {
         var apiKey = "api_key";
         var input = "input";
 
-        var request = new GoogleAiStudioCompletionRequest(List.of(input), GoogleAiStudioCompletionModelTests.createModel("model", apiKey));
+        var request = new GoogleAiStudioCompletionRequest(listOf(input), GoogleAiStudioCompletionModelTests.createModel("model", apiKey));
 
         var httpRequest = request.createHttpRequest();
         assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
@@ -54,7 +55,7 @@ public void testCreateRequest() throws IOException {
 
     public void testTruncate_ReturnsSameInstance() {
         var request = new GoogleAiStudioCompletionRequest(
-            List.of("input"),
+            listOf("input"),
             GoogleAiStudioCompletionModelTests.createModel("model", "api key")
         );
         var truncatedRequest = request.truncate();
@@ -64,10 +65,14 @@ public void testTruncate_ReturnsSameInstance() {
 
     public void testTruncationInfo_ReturnsNull() {
         var request = new GoogleAiStudioCompletionRequest(
-            List.of("input"),
+            listOf("input"),
             GoogleAiStudioCompletionModelTests.createModel("model", "api key")
         );
 
         assertNull(request.getTruncationInfo());
     }
+
+    private static DocumentsOnlyInput listOf(String... input) {
+        return new DocumentsOnlyInput(List.of(input));
+    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/StreamingInferenceTestUtils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/StreamingInferenceTestUtils.java
new file mode 100644
index 0000000000000..e0aef58c4f3b3
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/StreamingInferenceTestUtils.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.streaming;
+
+import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Deque;
+
+public class StreamingInferenceTestUtils {
+
+    public static Deque<ServerSentEvent> events(String... data) {
+        var item = new ArrayDeque<ServerSentEvent>();
+        Arrays.stream(data).map(datum -> new ServerSentEvent(ServerSentEventField.DATA, datum)).forEach(item::offer);
+        return item;
+    }
+
+    @SuppressWarnings("unchecked")
+    public static Matcher<Iterable<? extends StreamingChatCompletionResults.Result>> containsResults(String... results) {
+        Matcher<StreamingChatCompletionResults.Result>[] resultMatcher = Arrays.stream(results)
+            .map(StreamingChatCompletionResults.Result::new)
+            .map(Matchers::equalTo)
+            .toArray(Matcher[]::new);
+        return Matchers.contains(resultMatcher);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
index 7c8d1bbf9fb4d..f444719c730f5 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
@@ -57,6 +57,8 @@
 import org.elasticsearch.search.NestedDocuments;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.inference.DefaultElserFeatureFlag;
@@ -65,12 +67,16 @@
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.BiConsumer;
+import java.util.stream.Stream;
 
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_EMBEDDINGS_FIELD;
@@ -592,7 +598,7 @@ public void testSuccessfulParse() throws IOException {
                         generateNestedTermSparseVectorQuery(mapperService.mappingLookup().nestedLookup(), fieldName1, List.of("a")),
                         10
                     );
-                    assertEquals(1, topDocs.totalHits.value);
+                    assertEquals(1, topDocs.totalHits.value());
                     assertEquals(3, topDocs.scoreDocs[0].doc);
                 }
                 {
@@ -600,7 +606,7 @@ public void testSuccessfulParse() throws IOException {
                         generateNestedTermSparseVectorQuery(mapperService.mappingLookup().nestedLookup(), fieldName1, List.of("a", "b")),
                         10
                     );
-                    assertEquals(1, topDocs.totalHits.value);
+                    assertEquals(1, topDocs.totalHits.value());
                     assertEquals(3, topDocs.scoreDocs[0].doc);
                 }
                 {
@@ -608,7 +614,7 @@ public void testSuccessfulParse() throws IOException {
                         generateNestedTermSparseVectorQuery(mapperService.mappingLookup().nestedLookup(), fieldName2, List.of("d")),
                         10
                     );
-                    assertEquals(1, topDocs.totalHits.value);
+                    assertEquals(1, topDocs.totalHits.value());
                     assertEquals(3, topDocs.scoreDocs[0].doc);
                 }
                 {
@@ -616,7 +622,7 @@ public void testSuccessfulParse() throws IOException {
                         generateNestedTermSparseVectorQuery(mapperService.mappingLookup().nestedLookup(), fieldName2, List.of("z")),
                         10
                     );
-                    assertEquals(0, topDocs.totalHits.value);
+                    assertEquals(0, topDocs.totalHits.value());
                 }
             });
         }
@@ -795,6 +801,266 @@ public void testExistsQueryDenseVector() throws IOException {
         assertThat(existsQuery, instanceOf(ESToParentBlockJoinQuery.class));
     }
 
+    public void testInsertValueMapTraversal() throws IOException {
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("test", "value").endObject();
+
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            SemanticTextFieldMapper.insertValue("test", map, "value2");
+            assertThat(getMapValue(map, "test"), equalTo("value2"));
+            SemanticTextFieldMapper.insertValue("something.else", map, "something_else_value");
+            assertThat(getMapValue(map, "something\\.else"), equalTo("something_else_value"));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            builder.startObject("path1").startObject("path2").field("test", "value").endObject().endObject();
+            builder.endObject();
+
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            SemanticTextFieldMapper.insertValue("path1.path2.test", map, "value2");
+            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
+            SemanticTextFieldMapper.insertValue("path1.path2.test_me", map, "test_me_value");
+            assertThat(getMapValue(map, "path1.path2.test_me"), equalTo("test_me_value"));
+            SemanticTextFieldMapper.insertValue("path1.non_path2.test", map, "test_value");
+            assertThat(getMapValue(map, "path1.non_path2\\.test"), equalTo("test_value"));
+
+            SemanticTextFieldMapper.insertValue("path1.path2", map, Map.of("path3", "bar"));
+            assertThat(getMapValue(map, "path1.path2"), equalTo(Map.of("path3", "bar")));
+
+            SemanticTextFieldMapper.insertValue("path1", map, "baz");
+            assertThat(getMapValue(map, "path1"), equalTo("baz"));
+
+            SemanticTextFieldMapper.insertValue("path3.path4", map, Map.of("test", "foo"));
+            assertThat(getMapValue(map, "path3\\.path4"), equalTo(Map.of("test", "foo")));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            builder.startObject("path1").array("test", "value1", "value2").endObject();
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextFieldMapper.insertValue("path1.test", map, List.of("value3", "value4", "value5"));
+            assertThat(getMapValue(map, "path1.test"), equalTo(List.of("value3", "value4", "value5")));
+
+            SemanticTextFieldMapper.insertValue("path2.test", map, List.of("value6", "value7", "value8"));
+            assertThat(getMapValue(map, "path2\\.test"), equalTo(List.of("value6", "value7", "value8")));
+        }
+    }
+
+    public void testInsertValueListTraversal() throws IOException {
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1");
+                {
+                    builder.startArray("path2");
+                    builder.startObject().field("test", "value1").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            {
+                builder.startObject("path3");
+                {
+                    builder.startArray("path4");
+                    builder.startObject().field("test", "value1").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextFieldMapper.insertValue("path1.path2.test", map, "value2");
+            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
+            SemanticTextFieldMapper.insertValue("path1.path2.test2", map, "value3");
+            assertThat(getMapValue(map, "path1.path2.test2"), equalTo("value3"));
+            assertThat(getMapValue(map, "path1.path2"), equalTo(List.of(Map.of("test", "value2", "test2", "value3"))));
+
+            SemanticTextFieldMapper.insertValue("path3.path4.test", map, "value4");
+            assertThat(getMapValue(map, "path3.path4.test"), equalTo("value4"));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1");
+                {
+                    builder.startArray("path2");
+                    builder.startArray();
+                    builder.startObject().field("test", "value1").endObject();
+                    builder.endArray();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextFieldMapper.insertValue("path1.path2.test", map, "value2");
+            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
+            SemanticTextFieldMapper.insertValue("path1.path2.test2", map, "value3");
+            assertThat(getMapValue(map, "path1.path2.test2"), equalTo("value3"));
+            assertThat(getMapValue(map, "path1.path2"), equalTo(List.of(List.of(Map.of("test", "value2", "test2", "value3")))));
+        }
+    }
+
+    public void testInsertValueFieldsWithDots() throws IOException {
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("xxx.yyy", "value1").endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextFieldMapper.insertValue("xxx.yyy", map, "value2");
+            assertThat(getMapValue(map, "xxx\\.yyy"), equalTo("value2"));
+
+            SemanticTextFieldMapper.insertValue("xxx", map, "value3");
+            assertThat(getMapValue(map, "xxx"), equalTo("value3"));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1.path2");
+                {
+                    builder.startObject("path3.path4");
+                    builder.field("test", "value1");
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextFieldMapper.insertValue("path1.path2.path3.path4.test", map, "value2");
+            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4.test"), equalTo("value2"));
+
+            SemanticTextFieldMapper.insertValue("path1.path2.path3.path4.test2", map, "value3");
+            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4.test2"), equalTo("value3"));
+            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4"), equalTo(Map.of("test", "value2", "test2", "value3")));
+        }
+    }
+
+    public void testInsertValueAmbiguousPath() throws IOException {
+        // Mixed dotted object notation
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1.path2");
+                {
+                    builder.startObject("path3");
+                    builder.field("test1", "value1");
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            {
+                builder.startObject("path1");
+                {
+                    builder.startObject("path2.path3");
+                    builder.field("test2", "value2");
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
+
+            IllegalArgumentException ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test1", map, "value3")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test1] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test3", map, "value4")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test3] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            assertThat(map, equalTo(originalMap));
+        }
+
+        // traversal through lists
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1.path2");
+                {
+                    builder.startArray("path3");
+                    builder.startObject().field("test1", "value1").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            {
+                builder.startObject("path1");
+                {
+                    builder.startArray("path2.path3");
+                    builder.startObject().field("test2", "value2").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
+
+            IllegalArgumentException ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test1", map, "value3")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test1] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test3", map, "value4")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test3] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            assertThat(map, equalTo(originalMap));
+        }
+    }
+
+    public void testInsertValueCannotTraversePath() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        {
+            builder.startObject("path1");
+            {
+                builder.startArray("path2");
+                builder.startArray();
+                builder.startObject().field("test", "value1").endObject();
+                builder.endArray();
+                builder.endArray();
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        Map<String, Object> map = toSourceMap(Strings.toString(builder));
+        final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
+
+        IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> SemanticTextFieldMapper.insertValue("path1.path2.test.test2", map, "value2")
+        );
+        assertThat(
+            ex.getMessage(),
+            equalTo("Path [path1.path2.test] has value [value1] of type [String], which cannot be traversed into further")
+        );
+
+        assertThat(map, equalTo(originalMap));
+    }
+
     @Override
     protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) {
         // Until a doc is indexed, the query is rewritten as match no docs
@@ -875,4 +1141,68 @@ private static void assertSparseFeatures(LuceneDocument doc, String fieldName, i
         }
         assertThat(count, equalTo(expectedCount));
     }
+
+    private Map<String, Object> toSourceMap(String source) throws IOException {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            return parser.map();
+        }
+    }
+
+    private static Object getMapValue(Map<String, Object> map, String key) {
+        // Split the path on unescaped "." chars and then unescape the escaped "." chars
+        final String[] pathElements = Arrays.stream(key.split("(?<!\\\\)\\.")).map(k -> k.replace("\\.", ".")).toArray(String[]::new);
+
+        Object value = null;
+        Object nextLayer = map;
+        for (int i = 0; i < pathElements.length; i++) {
+            if (nextLayer instanceof Map<?, ?> nextMap) {
+                value = nextMap.get(pathElements[i]);
+            } else if (nextLayer instanceof List<?> nextList) {
+                final String pathElement = pathElements[i];
+                List<?> values = nextList.stream().flatMap(v -> {
+                    Stream.Builder<Object> streamBuilder = Stream.builder();
+                    if (v instanceof List<?> innerList) {
+                        traverseList(innerList, streamBuilder);
+                    } else {
+                        streamBuilder.add(v);
+                    }
+                    return streamBuilder.build();
+                }).filter(v -> v instanceof Map<?, ?>).map(v -> ((Map<?, ?>) v).get(pathElement)).filter(Objects::nonNull).toList();
+
+                if (values.isEmpty()) {
+                    return null;
+                } else if (values.size() > 1) {
+                    throw new AssertionError("List " + nextList + " contains multiple values for [" + pathElement + "]");
+                } else {
+                    value = values.getFirst();
+                }
+            } else if (nextLayer == null) {
+                break;
+            } else {
+                throw new AssertionError(
+                    "Path ["
+                        + String.join(".", Arrays.copyOfRange(pathElements, 0, i))
+                        + "] has value ["
+                        + value
+                        + "] of type ["
+                        + value.getClass().getSimpleName()
+                        + "], which cannot be traversed into further"
+                );
+            }
+
+            nextLayer = value;
+        }
+
+        return value;
+    }
+
+    private static void traverseList(List<?> list, Stream.Builder<Object> streamBuilder) {
+        for (Object value : list) {
+            if (value instanceof List<?> innerList) {
+                traverseList(innerList, streamBuilder);
+            } else {
+                streamBuilder.add(value);
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java
index d8c25fb5a6d88..779a98e023455 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
@@ -217,6 +218,11 @@ public String getWriteableName() {
         public TransportVersion getMinimalSupportedVersion() {
             return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests
         }
+
+        @Override
+        public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+            return TestTaskSettings.fromMap(new HashMap<>(newSettings));
+        }
     }
 
     public record TestSecretSettings(String apiKey) implements SecretSettings {
@@ -265,5 +271,10 @@ public String getWriteableName() {
         public TransportVersion getMinimalSupportedVersion() {
             return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests
         }
+
+        @Override
+        public SecretSettings newSecretSettings(Map<String, Object> newSecrets) {
+            return new TestSecretSettings(newSecrets.get("api_key").toString());
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java
index f54ce89183079..b8bcb766b53e1 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java
@@ -199,9 +199,9 @@ private void assertSparseEmbeddingLuceneQuery(Query query) {
         BooleanQuery innerBooleanQuery = (BooleanQuery) innerQuery;
         assertThat(innerBooleanQuery.clauses().size(), equalTo(queryTokenCount));
         innerBooleanQuery.forEach(c -> {
-            assertThat(c.getOccur(), equalTo(SHOULD));
-            assertThat(c.getQuery(), instanceOf(BoostQuery.class));
-            assertThat(((BoostQuery) c.getQuery()).getBoost(), equalTo(TOKEN_WEIGHT));
+            assertThat(c.occur(), equalTo(SHOULD));
+            assertThat(c.query(), instanceOf(BoostQuery.class));
+            assertThat(((BoostQuery) c.query()).getBoost(), equalTo(TOKEN_WEIGHT));
         });
     }
 
@@ -223,7 +223,7 @@ private Query assertOuterBooleanQuery(Query query) {
         List<BooleanClause> outerMustClauses = new ArrayList<>();
         List<BooleanClause> outerFilterClauses = new ArrayList<>();
         for (BooleanClause clause : outerBooleanQuery.clauses()) {
-            BooleanClause.Occur occur = clause.getOccur();
+            BooleanClause.Occur occur = clause.occur();
             if (occur == MUST) {
                 outerMustClauses.add(clause);
             } else if (occur == FILTER) {
@@ -236,7 +236,7 @@ private Query assertOuterBooleanQuery(Query query) {
         assertThat(outerMustClauses.size(), equalTo(1));
         assertThat(outerFilterClauses.size(), equalTo(1));
 
-        return outerMustClauses.get(0).getQuery();
+        return outerMustClauses.get(0).query();
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java
index 75c370fd4d3fb..409d62426949c 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.inference.InferenceService;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.search.SearchHit;
@@ -35,16 +36,16 @@
 import org.junit.Before;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -292,58 +293,30 @@ public void testStoreModel_ThrowsException_WhenFailureIsNotAVersionConflict() {
         );
     }
 
-    @SuppressWarnings("unchecked")
-    public void testDeepCopyDefaultConfig() {
-        {
-            var toCopy = new UnparsedModel("tocopy", randomFrom(TaskType.values()), "service-a", Map.of(), Map.of());
-            var copied = ModelRegistry.deepCopyDefaultConfig(toCopy);
-            assertThat(copied, not(sameInstance(toCopy)));
-            assertThat(copied.taskType(), is(toCopy.taskType()));
-            assertThat(copied.service(), is(toCopy.service()));
-            assertThat(copied.secrets(), not(sameInstance(toCopy.secrets())));
-            assertThat(copied.secrets(), is(toCopy.secrets()));
-            // Test copied is a modifiable map
-            copied.secrets().put("foo", "bar");
-
-            assertThat(copied.settings(), not(sameInstance(toCopy.settings())));
-            assertThat(copied.settings(), is(toCopy.settings()));
-            // Test copied is a modifiable map
-            copied.settings().put("foo", "bar");
-        }
+    public void testIdMatchedDefault() {
+        var defaultConfigIds = new ArrayList<InferenceService.DefaultConfigId>();
+        defaultConfigIds.add(new InferenceService.DefaultConfigId("foo", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class)));
+        defaultConfigIds.add(new InferenceService.DefaultConfigId("bar", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class)));
 
-        {
-            Map<String, Object> secretsMap = Map.of("secret", "value");
-            Map<String, Object> chunking = Map.of("strategy", "word");
-            Map<String, Object> task = Map.of("user", "name");
-            Map<String, Object> service = Map.of("num_threads", 1, "adaptive_allocations", Map.of("enabled", true));
-            Map<String, Object> settings = Map.of("chunking_settings", chunking, "service_settings", service, "task_settings", task);
-
-            var toCopy = new UnparsedModel("tocopy", randomFrom(TaskType.values()), "service-a", settings, secretsMap);
-            var copied = ModelRegistry.deepCopyDefaultConfig(toCopy);
-            assertThat(copied, not(sameInstance(toCopy)));
-
-            assertThat(copied.secrets(), not(sameInstance(toCopy.secrets())));
-            assertThat(copied.secrets(), is(toCopy.secrets()));
-            // Test copied is a modifiable map
-            copied.secrets().remove("secret");
-
-            assertThat(copied.settings(), not(sameInstance(toCopy.settings())));
-            assertThat(copied.settings(), is(toCopy.settings()));
-            // Test copied is a modifiable map
-            var chunkOut = (Map<String, Object>) copied.settings().get("chunking_settings");
-            assertThat(chunkOut, is(chunking));
-            chunkOut.remove("strategy");
-
-            var taskOut = (Map<String, Object>) copied.settings().get("task_settings");
-            assertThat(taskOut, is(task));
-            taskOut.remove("user");
-
-            var serviceOut = (Map<String, Object>) copied.settings().get("service_settings");
-            assertThat(serviceOut, is(service));
-            var adaptiveOut = (Map<String, Object>) serviceOut.remove("adaptive_allocations");
-            assertThat(adaptiveOut, is(Map.of("enabled", true)));
-            adaptiveOut.remove("enabled");
-        }
+        var matched = ModelRegistry.idMatchedDefault("bar", defaultConfigIds);
+        assertEquals(defaultConfigIds.get(1), matched.get());
+        matched = ModelRegistry.idMatchedDefault("baz", defaultConfigIds);
+        assertFalse(matched.isPresent());
+    }
+
+    public void testTaskTypeMatchedDefaults() {
+        var defaultConfigIds = new ArrayList<InferenceService.DefaultConfigId>();
+        defaultConfigIds.add(new InferenceService.DefaultConfigId("s1", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class)));
+        defaultConfigIds.add(new InferenceService.DefaultConfigId("s2", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class)));
+        defaultConfigIds.add(new InferenceService.DefaultConfigId("d1", TaskType.TEXT_EMBEDDING, mock(InferenceService.class)));
+        defaultConfigIds.add(new InferenceService.DefaultConfigId("c1", TaskType.COMPLETION, mock(InferenceService.class)));
+
+        var matched = ModelRegistry.taskTypeMatchedDefaults(TaskType.SPARSE_EMBEDDING, defaultConfigIds);
+        assertThat(matched, contains(defaultConfigIds.get(0), defaultConfigIds.get(1)));
+        matched = ModelRegistry.taskTypeMatchedDefaults(TaskType.TEXT_EMBEDDING, defaultConfigIds);
+        assertThat(matched, contains(defaultConfigIds.get(2)));
+        matched = ModelRegistry.taskTypeMatchedDefaults(TaskType.RERANK, defaultConfigIds);
+        assertThat(matched, empty());
     }
 
     public void testDuplicateDefaultIds() {
@@ -351,11 +324,15 @@ public void testDuplicateDefaultIds() {
         var registry = new ModelRegistry(client);
 
         var id = "my-inference";
+        var mockServiceA = mock(InferenceService.class);
+        when(mockServiceA.name()).thenReturn("service-a");
+        var mockServiceB = mock(InferenceService.class);
+        when(mockServiceB.name()).thenReturn("service-b");
 
-        registry.addDefaultConfiguration(new UnparsedModel(id, randomFrom(TaskType.values()), "service-a", Map.of(), Map.of()));
+        registry.addDefaultIds(new InferenceService.DefaultConfigId(id, randomFrom(TaskType.values()), mockServiceA));
         var ise = expectThrows(
             IllegalStateException.class,
-            () -> registry.addDefaultConfiguration(new UnparsedModel(id, randomFrom(TaskType.values()), "service-b", Map.of(), Map.of()))
+            () -> registry.addDefaultIds(new InferenceService.DefaultConfigId(id, randomFrom(TaskType.values()), mockServiceB))
         );
         assertThat(
             ise.getMessage(),
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java
index f23ea2aa414b2..7cfd231be39f3 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.inference.InferenceServiceResults;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
@@ -26,6 +25,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.test.ESTestCase.fail;
 import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.CoreMatchers.is;
 
@@ -47,7 +47,9 @@ public InferenceEventsAssertion hasFinishedStream() {
     }
 
     public InferenceEventsAssertion hasNoErrors() {
-        MatcherAssert.assertThat("Expected no errors from stream.", error, Matchers.nullValue());
+        if (error != null) {
+            fail(error, "Expected no errors from stream.");
+        }
         return this;
     }
 
@@ -66,7 +68,7 @@ public InferenceEventsAssertion hasErrorWithStatusCode(int statusCode) {
             }
             t = t.getCause();
         }
-        ESTestCase.fail(error, "Expected an underlying ElasticsearchStatusException.");
+        fail(error, "Expected an underlying ElasticsearchStatusException.");
         return this;
     }
 
@@ -79,7 +81,7 @@ public InferenceEventsAssertion hasErrorContaining(String message) {
             }
             t = t.getCause();
         }
-        ESTestCase.fail(error, "Expected exception to contain string: " + message);
+        fail(error, "Expected exception to contain string: " + message);
         return this;
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java
index ca48d5427d18b..e3df0f0b5a2e1 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java
@@ -605,6 +605,60 @@ public void testExtractRequiredPositiveIntegerLessThanOrEqualToMax_AddsErrorWhen
         assertThat(validation.validationErrors().get(1), is("[scope] does not contain the required setting [not_key]"));
     }
 
+    public void testExtractRequiredPositiveIntegerBetween_ReturnsValueWhenValueIsBetweenMinAndMax() {
+        var minValue = randomNonNegativeInt();
+        var maxValue = randomIntBetween(minValue + 2, minValue + 10);
+        testExtractRequiredPositiveIntegerBetween_Successful(minValue, maxValue, randomIntBetween(minValue + 1, maxValue - 1));
+    }
+
+    public void testExtractRequiredPositiveIntegerBetween_ReturnsValueWhenValueIsEqualToMin() {
+        var minValue = randomNonNegativeInt();
+        var maxValue = randomIntBetween(minValue + 1, minValue + 10);
+        testExtractRequiredPositiveIntegerBetween_Successful(minValue, maxValue, minValue);
+    }
+
+    public void testExtractRequiredPositiveIntegerBetween_ReturnsValueWhenValueIsEqualToMax() {
+        var minValue = randomNonNegativeInt();
+        var maxValue = randomIntBetween(minValue + 1, minValue + 10);
+        testExtractRequiredPositiveIntegerBetween_Successful(minValue, maxValue, maxValue);
+    }
+
+    private void testExtractRequiredPositiveIntegerBetween_Successful(int minValue, int maxValue, int actualValue) {
+        var validation = new ValidationException();
+        validation.addValidationError("previous error");
+        Map<String, Object> map = modifiableMap(Map.of("key", actualValue));
+        var parsedInt = ServiceUtils.extractRequiredPositiveIntegerBetween(map, "key", minValue, maxValue, "scope", validation);
+
+        assertThat(validation.validationErrors(), hasSize(1));
+        assertNotNull(parsedInt);
+        assertThat(parsedInt, is(actualValue));
+        assertTrue(map.isEmpty());
+    }
+
+    public void testExtractRequiredIntBetween_AddsErrorForValueBelowMin() {
+        var minValue = randomNonNegativeInt();
+        var maxValue = randomIntBetween(minValue, minValue + 10);
+        testExtractRequiredIntBetween_Unsuccessful(minValue, maxValue, minValue - 1);
+    }
+
+    public void testExtractRequiredIntBetween_AddsErrorForValueAboveMax() {
+        var minValue = randomNonNegativeInt();
+        var maxValue = randomIntBetween(minValue, minValue + 10);
+        testExtractRequiredIntBetween_Unsuccessful(minValue, maxValue, maxValue + 1);
+    }
+
+    private void testExtractRequiredIntBetween_Unsuccessful(int minValue, int maxValue, int actualValue) {
+        var validation = new ValidationException();
+        validation.addValidationError("previous error");
+        Map<String, Object> map = modifiableMap(Map.of("key", actualValue));
+        var parsedInt = ServiceUtils.extractRequiredPositiveIntegerBetween(map, "key", minValue, maxValue, "scope", validation);
+
+        assertThat(validation.validationErrors(), hasSize(2));
+        assertNull(parsedInt);
+        assertTrue(map.isEmpty());
+        assertThat(validation.validationErrors().get(1), containsString("Invalid value"));
+    }
+
     public void testExtractOptionalEnum_ReturnsNull_WhenFieldDoesNotExist() {
         var validation = new ValidationException();
         Map<String, Object> map = modifiableMap(Map.of("key", "value"));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java
index c48d57cf3e03b..7acba78b3066b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java
@@ -7,16 +7,17 @@
 
 package org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion;
 
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionTaskSettings.PARAMETERS;
 import static org.hamcrest.Matchers.is;
 
 public class AlibabaCloudSearchCompletionTaskSettingsTests extends AbstractWireSerializingTestCase<
@@ -34,10 +35,20 @@ public void testFromMap() {
         );
     }
 
-    public void testIsEmpty() {
-        var randomSettings = createRandom();
-        var stringRep = Strings.toString(randomSettings);
-        assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.getParameters() != null) {
+            newSettingsMap.put(PARAMETERS, newSettings.getParameters());
+        }
+        AlibabaCloudSearchCompletionTaskSettings updatedSettings = (AlibabaCloudSearchCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(Collections.unmodifiableMap(newSettingsMap));
+        if (newSettings.getParameters() == null) {
+            assertEquals(initialSettings.getParameters(), updatedSettings.getParameters());
+        } else {
+            assertEquals(newSettings.getParameters(), updatedSettings.getParameters());
+        }
     }
 
     @Override
@@ -60,7 +71,7 @@ public static Map<String, Object> getTaskSettingsMap(@Nullable Map<String, Objec
         var map = new HashMap<String, Object>();
 
         if (params != null) {
-            map.put(AlibabaCloudSearchCompletionTaskSettings.PARAMETERS, params);
+            map.put(PARAMETERS, params);
         }
 
         return map;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java
index 9e75a2f475051..4b558949fdc4a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java
@@ -15,10 +15,12 @@
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithIngestAndSearch;
+import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE;
 import static org.hamcrest.Matchers.is;
 
 public class AlibabaCloudSearchEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase<
@@ -31,13 +33,27 @@ public static AlibabaCloudSearchEmbeddingsTaskSettings createRandom() {
 
     public void testFromMap() {
         MatcherAssert.assertThat(
-            AlibabaCloudSearchEmbeddingsTaskSettings.fromMap(
-                new HashMap<>(Map.of(AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE, "ingest"))
-            ),
+            AlibabaCloudSearchEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(INPUT_TYPE, "ingest"))),
             is(new AlibabaCloudSearchEmbeddingsTaskSettings(InputType.INGEST))
         );
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.getInputType() != null) {
+            newSettingsMap.put(INPUT_TYPE, newSettings.getInputType().toString());
+        }
+        AlibabaCloudSearchEmbeddingsTaskSettings updatedSettings = (AlibabaCloudSearchEmbeddingsTaskSettings) initialSettings
+            .updatedTaskSettings(Collections.unmodifiableMap(newSettingsMap));
+        if (newSettings.getInputType() == null) {
+            assertEquals(initialSettings.getInputType(), updatedSettings.getInputType());
+        } else {
+            assertEquals(newSettings.getInputType(), updatedSettings.getInputType());
+        }
+    }
+
     public void testFromMap_WhenInputTypeIsNull() {
         InputType inputType = null;
         MatcherAssert.assertThat(
@@ -72,7 +88,7 @@ public static Map<String, Object> getTaskSettingsMap(@Nullable InputType inputTy
         var map = new HashMap<String, Object>();
 
         if (inputType != null) {
-            map.put(AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE, inputType.toString());
+            map.put(INPUT_TYPE, inputType.toString());
         }
 
         return map;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java
index 2c134c6765078..fa78b24d1a4bb 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java
@@ -19,6 +19,8 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithIngestAndSearch;
+import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE;
+import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings.RETURN_TOKEN;
 import static org.hamcrest.Matchers.is;
 
 public class AlibabaCloudSearchSparseTaskSettingsTests extends AbstractWireSerializingTestCase<AlibabaCloudSearchSparseTaskSettings> {
@@ -31,11 +33,33 @@ public static AlibabaCloudSearchSparseTaskSettings createRandom() {
 
     public void testFromMap() {
         MatcherAssert.assertThat(
-            AlibabaCloudSearchSparseTaskSettings.fromMap(new HashMap<>(Map.of(AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE, "ingest"))),
+            AlibabaCloudSearchSparseTaskSettings.fromMap(new HashMap<>(Map.of(INPUT_TYPE, "ingest"))),
             is(new AlibabaCloudSearchSparseTaskSettings(InputType.INGEST, null))
         );
     }
 
+    public void testUpdatedTaskSettings() {
+        {
+            var initialSettings = createRandom();
+            var newSettings = createRandom();
+            AlibabaCloudSearchSparseTaskSettings updatedSettings = (AlibabaCloudSearchSparseTaskSettings) initialSettings
+                .updatedTaskSettings(Map.of(RETURN_TOKEN, newSettings.isReturnToken()));
+        }
+        {
+            var initialSettings = createRandom();
+            var newSettings = createRandom();
+            AlibabaCloudSearchSparseTaskSettings updatedSettings = (AlibabaCloudSearchSparseTaskSettings) initialSettings
+                .updatedTaskSettings(
+                    Map.of(
+                        INPUT_TYPE,
+                        newSettings.getInputType() == null ? InputType.SEARCH.toString() : newSettings.getInputType().toString(),
+                        RETURN_TOKEN,
+                        newSettings.isReturnToken()
+                    )
+                );
+        }
+    }
+
     public void testIsEmpty() {
         var randomSettings = createRandom();
         var stringRep = Strings.toString(randomSettings);
@@ -69,11 +93,11 @@ public static Map<String, Object> getTaskSettingsMap(@Nullable InputType inputTy
         var map = new HashMap<String, Object>();
 
         if (inputType != null) {
-            map.put(AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE, inputType.toString());
+            map.put(INPUT_TYPE, inputType.toString());
         }
 
         if (returnToken != null) {
-            map.put(AlibabaCloudSearchSparseTaskSettings.RETURN_TOKEN, returnToken);
+            map.put(RETURN_TOKEN, returnToken);
         }
 
         return map;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java
index 904851842a6c8..88aebd2d9d42b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java
@@ -29,6 +29,17 @@
 
 public class AmazonBedrockSecretSettingsTests extends AbstractBWCWireSerializationTestCase<AmazonBedrockSecretSettings> {
 
+    public void testNewSecretSettings() {
+        AmazonBedrockSecretSettings initialSettings = createRandom();
+        AmazonBedrockSecretSettings newSettings = createRandom();
+
+        AmazonBedrockSecretSettings finalSettings = (AmazonBedrockSecretSettings) initialSettings.newSecretSettings(
+            Map.of(ACCESS_KEY_FIELD, newSettings.accessKey.toString(), SECRET_KEY_FIELD, newSettings.secretKey.toString())
+        );
+
+        assertEquals(newSettings, finalSettings);
+    }
+
     public void testIt_CreatesSettings_ReturnsNullFromMap_null() {
         var secrets = AmazonBedrockSecretSettings.fromMap(null);
         assertNull(secrets);
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java
index 9c746e7c2aed9..06c5a68987a9e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.inference.services.amazonbedrock;
 
+import software.amazon.awssdk.services.bedrockruntime.model.BedrockRuntimeException;
+
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
@@ -33,7 +35,6 @@
 import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.inference.Utils;
 import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockMockRequestSender;
-import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.Sender;
 import org.elasticsearch.xpack.inference.services.ServiceComponentsTests;
@@ -1265,12 +1266,19 @@ public void testInfer_UnauthorizedResponse() throws IOException {
         var factory = mock(HttpRequestSender.Factory.class);
         when(factory.createSender()).thenReturn(sender);
 
-        var amazonBedrockFactory = new AmazonBedrockRequestSender.Factory(
+        var amazonBedrockFactory = new AmazonBedrockMockRequestSender.Factory(
             ServiceComponentsTests.createWithSettings(threadPool, Settings.EMPTY),
             mockClusterServiceEmpty()
         );
 
-        try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) {
+        try (
+            var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool));
+            var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender()
+        ) {
+            requestSender.enqueue(
+                BedrockRuntimeException.builder().message("The security token included in the request is invalid").build()
+            );
+
             var model = AmazonBedrockEmbeddingsModelTests.createModel(
                 "id",
                 "us-east-1",
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java
index 69dd3b1e6257b..adbf2c66ca0e2 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java
@@ -38,6 +38,68 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void updatedTaskSettings_WithEmptyMap_ReturnsSameSettings() {
+        var initialSettings = createRandom();
+        AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(Map.of());
+        assertEquals(initialSettings, updatedSettings);
+    }
+
+    public void updatedTaskSettings_WithNewTemperature_ReturnsUpdatedSettings() {
+        var initialSettings = createRandom();
+        Map<String, Object> newSettings = Map.of(TEMPERATURE_FIELD, 0.7);
+        AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(newSettings);
+        assertEquals(0.7, (double) updatedSettings.temperature(), 0.001);
+        assertEquals(initialSettings.topP(), updatedSettings.topP());
+        assertEquals(initialSettings.topK(), updatedSettings.topK());
+        assertEquals(initialSettings.maxNewTokens(), updatedSettings.maxNewTokens());
+    }
+
+    public void updatedTaskSettings_WithNewTopP_ReturnsUpdatedSettings() {
+        var initialSettings = createRandom();
+        Map<String, Object> newSettings = Map.of(TOP_P_FIELD, 0.8);
+        AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(newSettings);
+        assertEquals(0.8, (double) updatedSettings.topP(), 0.001);
+        assertEquals(initialSettings.temperature(), updatedSettings.temperature());
+        assertEquals(initialSettings.topK(), updatedSettings.topK());
+        assertEquals(initialSettings.maxNewTokens(), updatedSettings.maxNewTokens());
+    }
+
+    public void updatedTaskSettings_WithNewTopK_ReturnsUpdatedSettings() {
+        var initialSettings = createRandom();
+        Map<String, Object> newSettings = Map.of(TOP_K_FIELD, 0.9);
+        AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(newSettings);
+        assertEquals(0.9, (double) updatedSettings.topK(), 0.001);
+        assertEquals(initialSettings.temperature(), updatedSettings.temperature());
+        assertEquals(initialSettings.topP(), updatedSettings.topP());
+        assertEquals(initialSettings.maxNewTokens(), updatedSettings.maxNewTokens());
+    }
+
+    public void updatedTaskSettings_WithNewMaxNewTokens_ReturnsUpdatedSettings() {
+        var initialSettings = createRandom();
+        Map<String, Object> newSettings = Map.of(MAX_NEW_TOKENS_FIELD, 256);
+        AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(newSettings);
+        assertEquals(256, (double) updatedSettings.maxNewTokens(), 0.001);
+        assertEquals(initialSettings.temperature(), updatedSettings.temperature());
+        assertEquals(initialSettings.topP(), updatedSettings.topP());
+        assertEquals(initialSettings.topK(), updatedSettings.topK());
+    }
+
+    public void updatedTaskSettings_WithMultipleNewValues_ReturnsUpdatedSettings() {
+        var initialSettings = createRandom();
+        Map<String, Object> newSettings = Map.of(TEMPERATURE_FIELD, 0.7, TOP_P_FIELD, 0.8, TOP_K_FIELD, 0.9, MAX_NEW_TOKENS_FIELD, 256);
+        AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(newSettings);
+        assertEquals(0.7, (double) updatedSettings.temperature(), 0.001);
+        assertEquals(0.8, (double) updatedSettings.topP(), 0.001);
+        assertEquals(0.9, (double) updatedSettings.topK(), 0.001);
+        assertEquals(256, (int) updatedSettings.maxNewTokens(), 0.001);
+    }
+
     public void testFromMap_AllValues() {
         var taskMap = getChatCompletionTaskSettingsMap(1.0, 0.5, 0.6, 512);
         assertEquals(
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java
index 8adf75b4c0a81..48277112d9306 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java
@@ -532,7 +532,6 @@ public void testInfer_SendsCompletionRequest() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest() throws Exception {
         String responseJson = """
             data: {"type": "message_start", "message": {"model": "claude, probably"}}
@@ -578,7 +577,6 @@ private InferenceServiceResults streamChatCompletion() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest_ErrorResponse() throws Exception {
         String responseJson = """
             data: {"type": "error", "error": {"type": "request_too_large", "message": "blah"}}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java
index e00de80e8709e..5f6823770345f 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java
@@ -24,6 +24,25 @@
 
 public class AnthropicChatCompletionTaskSettingsTests extends AbstractBWCWireSerializationTestCase<AnthropicChatCompletionTaskSettings> {
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        AnthropicChatCompletionTaskSettings updatedSettings = (AnthropicChatCompletionTaskSettings) initialSettings.updatedTaskSettings(
+            Map.of(
+                AnthropicServiceFields.MAX_TOKENS,
+                newSettings.maxTokens(),
+                AnthropicServiceFields.TEMPERATURE_FIELD,
+                newSettings.temperature(),
+                AnthropicServiceFields.TOP_P_FIELD,
+                newSettings.topP(),
+                AnthropicServiceFields.TOP_K_FIELD,
+                newSettings.topK()
+            )
+        );
+
+        assertEquals(newSettings, updatedSettings);
+    }
+
     public static Map<String, Object> getChatCompletionTaskSettingsMap(
         @Nullable Integer maxTokens,
         @Nullable Double temperature,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
index 44b0d17d9b448..e85edf573ba96 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
@@ -1308,7 +1308,6 @@ public void testInfer_UnauthorisedResponse() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest() throws Exception {
         String responseJson = """
             data: {\
@@ -1364,7 +1363,6 @@ private InferenceServiceResults streamChatCompletion() throws IOException, URISy
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest_ErrorResponse() throws Exception {
         String responseJson = """
             {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java
index 8d7dcf1ef5170..21c1a233348fe 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java
@@ -19,6 +19,7 @@
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -38,6 +39,30 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        var settingsMap = new HashMap<String, Object>();
+        if (newSettings.doSample() != null) settingsMap.put(DO_SAMPLE_FIELD, newSettings.doSample());
+        if (newSettings.temperature() != null) settingsMap.put(TEMPERATURE_FIELD, newSettings.temperature());
+        if (newSettings.topP() != null) settingsMap.put(TOP_P_FIELD, newSettings.topP());
+        if (newSettings.maxNewTokens() != null) settingsMap.put(MAX_NEW_TOKENS_FIELD, newSettings.maxNewTokens());
+
+        AzureAiStudioChatCompletionTaskSettings updatedSettings = (AzureAiStudioChatCompletionTaskSettings) initialSettings
+            .updatedTaskSettings(Collections.unmodifiableMap(settingsMap));
+
+        assertEquals(
+            newSettings.temperature() == null ? initialSettings.temperature() : newSettings.temperature(),
+            updatedSettings.temperature()
+        );
+        assertEquals(newSettings.topP() == null ? initialSettings.topP() : newSettings.topP(), updatedSettings.topP());
+        assertEquals(newSettings.doSample() == null ? initialSettings.doSample() : newSettings.doSample(), updatedSettings.doSample());
+        assertEquals(
+            newSettings.maxNewTokens() == null ? initialSettings.maxNewTokens() : newSettings.maxNewTokens(),
+            updatedSettings.maxNewTokens()
+        );
+    }
+
     public void testFromMap_AllValues() {
         var taskMap = getTaskSettingsMap(1.0, 2.0, true, 512);
         assertEquals(
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java
index 4b6b38bd15c0d..cdfde5fcb09c9 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java
@@ -20,6 +20,7 @@
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -32,6 +33,23 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.user() != null) {
+            newSettingsMap.put(AzureAiStudioConstants.USER_FIELD, newSettings.user());
+        }
+        AzureAiStudioEmbeddingsTaskSettings updatedSettings = (AzureAiStudioEmbeddingsTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        if (newSettings.user() == null) {
+            assertEquals(initialSettings.user(), updatedSettings.user());
+        } else {
+            assertEquals(newSettings.user(), updatedSettings.user());
+        }
+    }
+
     public void testFromMap_WithUser() {
         assertEquals(
             new AzureAiStudioEmbeddingsTaskSettings("user"),
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java
index e08365e7ca3bf..dbbf90054a55b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java
@@ -30,7 +30,31 @@
 public class AzureOpenAiSecretSettingsTests extends AbstractBWCWireSerializationTestCase<AzureOpenAiSecretSettings> {
 
     public static AzureOpenAiSecretSettings createRandom() {
-        return new AzureOpenAiSecretSettings(randomSecureStringOfLength(15), randomSecureStringOfLength(15));
+        boolean isApiKeyNotEntraId = randomBoolean();
+        return new AzureOpenAiSecretSettings(
+            isApiKeyNotEntraId ? randomSecureStringOfLength(15) : null,
+            isApiKeyNotEntraId == false ? randomSecureStringOfLength(15) : null
+        );
+    }
+
+    public void testNewSecretSettingsApiKey() {
+        AzureOpenAiSecretSettings initialSettings = createRandom();
+        AzureOpenAiSecretSettings newSettings = new AzureOpenAiSecretSettings(randomSecureStringOfLength(15), null);
+        AzureOpenAiSecretSettings finalSettings = (AzureOpenAiSecretSettings) initialSettings.newSecretSettings(
+            Map.of(API_KEY, newSettings.apiKey().toString())
+        );
+
+        assertEquals(newSettings, finalSettings);
+    }
+
+    public void testNewSecretSettingsEntraId() {
+        AzureOpenAiSecretSettings initialSettings = createRandom();
+        AzureOpenAiSecretSettings newSettings = new AzureOpenAiSecretSettings(null, randomSecureStringOfLength(15));
+        AzureOpenAiSecretSettings finalSettings = (AzureOpenAiSecretSettings) initialSettings.newSecretSettings(
+            Map.of(ENTRA_ID, newSettings.entraId().toString())
+        );
+
+        assertEquals(newSettings, finalSettings);
     }
 
     public void testFromMap_ApiKey_Only() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
index 900b666c0b8fb..3408fc358cac0 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
@@ -1425,7 +1425,6 @@ private void testChunkedInfer(AzureOpenAiEmbeddingsModel model) throws IOExcepti
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest() throws Exception {
         String responseJson = """
             data: {\
@@ -1484,7 +1483,6 @@ private InferenceServiceResults streamChatCompletion() throws IOException, URISy
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest_ErrorResponse() throws Exception {
         String responseJson = """
             {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java
index 8e8d9c4f92800..9d77abfe6d512 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java
@@ -38,6 +38,16 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        AzureOpenAiCompletionTaskSettings updatedSettings = (AzureOpenAiCompletionTaskSettings) initialSettings.updatedTaskSettings(
+            newSettings.user() == null ? Map.of() : Map.of(AzureOpenAiServiceFields.USER, newSettings.user())
+        );
+
+        assertEquals(newSettings.user() == null ? initialSettings.user() : newSettings.user(), updatedSettings.user());
+    }
+
     public void testFromMap_WithUser() {
         var user = "user";
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java
index 72a063af37b90..4df9f2f6bcce0 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java
@@ -12,13 +12,13 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.USER;
 import static org.hamcrest.Matchers.is;
 
 public class AzureOpenAiEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase<AzureOpenAiEmbeddingsTaskSettings> {
@@ -41,17 +41,31 @@ public static AzureOpenAiEmbeddingsTaskSettings createRandom() {
         return new AzureOpenAiEmbeddingsTaskSettings(user);
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        AzureOpenAiEmbeddingsTaskSettings updatedSettings = (AzureOpenAiEmbeddingsTaskSettings) initialSettings.updatedTaskSettings(
+            newSettings.user() == null ? Map.of() : Map.of(USER, newSettings.user())
+        );
+
+        if (newSettings.user() == null) {
+            assertEquals(initialSettings.user(), updatedSettings.user());
+        } else {
+            assertEquals(newSettings.user(), updatedSettings.user());
+        }
+    }
+
     public void testFromMap_WithUser() {
         assertEquals(
             new AzureOpenAiEmbeddingsTaskSettings("user"),
-            AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user")))
+            AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user")))
         );
     }
 
     public void testFromMap_UserIsEmptyString() {
         var thrownException = expectThrows(
             ValidationException.class,
-            () -> AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "")))
+            () -> AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, "")))
         );
 
         MatcherAssert.assertThat(
@@ -66,7 +80,7 @@ public void testFromMap_MissingUser_DoesNotThrowException() {
     }
 
     public void testOverrideWith_KeepsOriginalValuesWithOverridesAreNull() {
-        var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user")));
+        var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user")));
 
         var overriddenTaskSettings = AzureOpenAiEmbeddingsTaskSettings.of(
             taskSettings,
@@ -76,11 +90,9 @@ public void testOverrideWith_KeepsOriginalValuesWithOverridesAreNull() {
     }
 
     public void testOverrideWith_UsesOverriddenSettings() {
-        var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user")));
+        var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user")));
 
-        var requestTaskSettings = AzureOpenAiEmbeddingsRequestTaskSettings.fromMap(
-            new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user2"))
-        );
+        var requestTaskSettings = AzureOpenAiEmbeddingsRequestTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user2")));
 
         var overriddenTaskSettings = AzureOpenAiEmbeddingsTaskSettings.of(taskSettings, requestTaskSettings);
         MatcherAssert.assertThat(overriddenTaskSettings, is(new AzureOpenAiEmbeddingsTaskSettings("user2")));
@@ -105,7 +117,7 @@ public static Map<String, Object> getAzureOpenAiRequestTaskSettingsMap(@Nullable
         var map = new HashMap<String, Object>();
 
         if (user != null) {
-            map.put(AzureOpenAiServiceFields.USER, user);
+            map.put(USER, user);
         }
 
         return map;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
index cf114db45619f..758c38166778b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
@@ -1635,7 +1635,6 @@ public void testDefaultSimilarity() {
         assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest() throws Exception {
         String responseJson = """
             {"event_type":"text-generation", "text":"hello"}
@@ -1669,7 +1668,6 @@ private InferenceServiceResults streamChatCompletion() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest_ErrorResponse() throws Exception {
         String responseJson = """
             { "event_type":"stream-end", "finish_reason":"ERROR", "response":{ "text": "how dare you" } }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java
index 90c9b032465c6..3df8fcaf5d6b8 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Locale;
@@ -43,6 +44,31 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.getInputType() != null) {
+            newSettingsMap.put(CohereEmbeddingsTaskSettings.INPUT_TYPE, newSettings.getInputType().toString());
+        }
+        if (newSettings.getTruncation() != null) {
+            newSettingsMap.put(CohereServiceFields.TRUNCATE, newSettings.getTruncation().toString());
+        }
+        CohereEmbeddingsTaskSettings updatedSettings = (CohereEmbeddingsTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        if (newSettings.getInputType() == null) {
+            assertEquals(initialSettings.getInputType(), updatedSettings.getInputType());
+        } else {
+            assertEquals(newSettings.getInputType(), updatedSettings.getInputType());
+        }
+        if (newSettings.getTruncation() == null) {
+            assertEquals(initialSettings.getTruncation(), updatedSettings.getTruncation());
+        } else {
+            assertEquals(newSettings.getTruncation(), updatedSettings.getTruncation());
+        }
+    }
+
     public void testFromMap_CreatesEmptySettings_WhenAllFieldsAreNull() {
         MatcherAssert.assertThat(
             CohereEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of())),
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java
index a729ac8e225b5..e3401b74017f2 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java
@@ -78,7 +78,7 @@ protected CohereRerankServiceSettings mutateInstance(CohereRerankServiceSettings
 
     @Override
     protected CohereRerankServiceSettings mutateInstanceForVersion(CohereRerankServiceSettings instance, TransportVersion version) {
-        if (version.before(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) {
+        if (version.before(TransportVersions.V_8_15_0)) {
             // We always default to the same rate limit settings, if a node is on a version before rate limits were introduced
             return new CohereRerankServiceSettings(instance.uri(), instance.modelId(), CohereServiceSettings.DEFAULT_RATE_LIMIT_SETTINGS);
         }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettingsTests.java
new file mode 100644
index 0000000000000..6924ee05ecbb8
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettingsTests.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.cohere.rerank;
+
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class CohereRerankTaskSettingsTests extends AbstractWireSerializingTestCase<CohereRerankTaskSettings> {
+
+    public static CohereRerankTaskSettings createRandom() {
+        var returnDocuments = randomBoolean() ? randomBoolean() : null;
+        var topNDocsOnly = randomBoolean() ? randomIntBetween(1, 10) : null;
+        var maxChunksPerDoc = randomBoolean() ? randomIntBetween(1, 20) : null;
+
+        return new CohereRerankTaskSettings(topNDocsOnly, returnDocuments, maxChunksPerDoc);
+    }
+
+    public void testFromMap_WithValidValues_ReturnsSettings() {
+        Map<String, Object> taskMap = Map.of(
+            CohereRerankTaskSettings.RETURN_DOCUMENTS,
+            true,
+            CohereRerankTaskSettings.TOP_N_DOCS_ONLY,
+            5,
+            CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC,
+            10
+        );
+        var settings = CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap));
+        assertTrue(settings.getReturnDocuments());
+        assertEquals(5, settings.getTopNDocumentsOnly().intValue());
+        assertEquals(10, settings.getMaxChunksPerDoc().intValue());
+    }
+
+    public void testFromMap_WithNullValues_ReturnsSettingsWithNulls() {
+        var settings = CohereRerankTaskSettings.fromMap(Map.of());
+        assertNull(settings.getReturnDocuments());
+        assertNull(settings.getTopNDocumentsOnly());
+        assertNull(settings.getMaxChunksPerDoc());
+    }
+
+    public void testFromMap_WithInvalidReturnDocuments_ThrowsValidationException() {
+        Map<String, Object> taskMap = Map.of(
+            CohereRerankTaskSettings.RETURN_DOCUMENTS,
+            "invalid",
+            CohereRerankTaskSettings.TOP_N_DOCS_ONLY,
+            5,
+            CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC,
+            10
+        );
+        var thrownException = expectThrows(ValidationException.class, () -> CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap)));
+        assertThat(thrownException.getMessage(), containsString("field [return_documents] is not of the expected type"));
+    }
+
+    public void testFromMap_WithInvalidTopNDocsOnly_ThrowsValidationException() {
+        Map<String, Object> taskMap = Map.of(
+            CohereRerankTaskSettings.RETURN_DOCUMENTS,
+            true,
+            CohereRerankTaskSettings.TOP_N_DOCS_ONLY,
+            "invalid",
+            CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC,
+            10
+        );
+        var thrownException = expectThrows(ValidationException.class, () -> CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap)));
+        assertThat(thrownException.getMessage(), containsString("field [top_n] is not of the expected type"));
+    }
+
+    public void testFromMap_WithInvalidMaxChunksPerDoc_ThrowsValidationException() {
+        Map<String, Object> taskMap = Map.of(
+            CohereRerankTaskSettings.RETURN_DOCUMENTS,
+            true,
+            CohereRerankTaskSettings.TOP_N_DOCS_ONLY,
+            5,
+            CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC,
+            "invalid"
+        );
+        var thrownException = expectThrows(ValidationException.class, () -> CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap)));
+        assertThat(thrownException.getMessage(), containsString("field [max_chunks_per_doc] is not of the expected type"));
+    }
+
+    public void UpdatedTaskSettings_WithEmptyMap_ReturnsSameSettings() {
+        var initialSettings = new CohereRerankTaskSettings(5, true, 10);
+        CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(Map.of());
+        assertEquals(initialSettings, updatedSettings);
+    }
+
+    public void testUpdatedTaskSettings_WithNewReturnDocuments_ReturnsUpdatedSettings() {
+        var initialSettings = new CohereRerankTaskSettings(5, true, 10);
+        Map<String, Object> newSettings = Map.of(CohereRerankTaskSettings.RETURN_DOCUMENTS, false);
+        CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings);
+        assertFalse(updatedSettings.getReturnDocuments());
+        assertEquals(initialSettings.getTopNDocumentsOnly(), updatedSettings.getTopNDocumentsOnly());
+        assertEquals(initialSettings.getMaxChunksPerDoc(), updatedSettings.getMaxChunksPerDoc());
+    }
+
+    public void testUpdatedTaskSettings_WithNewTopNDocsOnly_ReturnsUpdatedSettings() {
+        var initialSettings = new CohereRerankTaskSettings(5, true, 10);
+        Map<String, Object> newSettings = Map.of(CohereRerankTaskSettings.TOP_N_DOCS_ONLY, 7);
+        CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings);
+        assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue());
+        assertEquals(initialSettings.getReturnDocuments(), updatedSettings.getReturnDocuments());
+        assertEquals(initialSettings.getMaxChunksPerDoc(), updatedSettings.getMaxChunksPerDoc());
+    }
+
+    public void testUpdatedTaskSettings_WithNewMaxChunksPerDoc_ReturnsUpdatedSettings() {
+        var initialSettings = new CohereRerankTaskSettings(5, true, 10);
+        Map<String, Object> newSettings = Map.of(CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC, 15);
+        CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings);
+        assertEquals(15, updatedSettings.getMaxChunksPerDoc().intValue());
+        assertEquals(initialSettings.getReturnDocuments(), updatedSettings.getReturnDocuments());
+        assertEquals(initialSettings.getTopNDocumentsOnly(), updatedSettings.getTopNDocumentsOnly());
+    }
+
+    public void testUpdatedTaskSettings_WithMultipleNewValues_ReturnsUpdatedSettings() {
+        var initialSettings = new CohereRerankTaskSettings(5, true, 10);
+        Map<String, Object> newSettings = Map.of(
+            CohereRerankTaskSettings.RETURN_DOCUMENTS,
+            false,
+            CohereRerankTaskSettings.TOP_N_DOCS_ONLY,
+            7,
+            CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC,
+            15
+        );
+        CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings);
+        assertFalse(updatedSettings.getReturnDocuments());
+        assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue());
+        assertEquals(15, updatedSettings.getMaxChunksPerDoc().intValue());
+    }
+
+    @Override
+    protected Writeable.Reader<CohereRerankTaskSettings> instanceReader() {
+        return CohereRerankTaskSettings::new;
+    }
+
+    @Override
+    protected CohereRerankTaskSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected CohereRerankTaskSettings mutateInstance(CohereRerankTaskSettings instance) throws IOException {
+        return randomValueOtherThan(instance, CohereRerankTaskSettingsTests::createRandom);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java
index 72e6daa911c1d..4207896fc54f3 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java
@@ -15,7 +15,9 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.sameInstance;
@@ -28,6 +30,23 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.returnDocuments() != null) {
+            newSettingsMap.put(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, newSettings.returnDocuments());
+        }
+        CustomElandRerankTaskSettings updatedSettings = (CustomElandRerankTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        if (newSettings.returnDocuments() == null) {
+            assertEquals(initialSettings.returnDocuments(), updatedSettings.returnDocuments());
+        } else {
+            assertEquals(newSettings.returnDocuments(), updatedSettings.returnDocuments());
+        }
+    }
+
     public void testDefaultsFromMap_MapIsNull_ReturnsDefaultSettings() {
         var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(null);
 
@@ -69,18 +88,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             {"return_documents":true}"""));
     }
 
-    public void testToXContent_DoesNotWriteReturnDocuments_IfNull() throws IOException {
-        Boolean bool = null;
-        var serviceSettings = new CustomElandRerankTaskSettings(bool);
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        serviceSettings.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-
-        assertThat(xContentResult, is("""
-            {}"""));
-    }
-
     public void testOf_PrefersNonNullRequestTaskSettings() {
         var originalSettings = new CustomElandRerankTaskSettings(Boolean.FALSE);
         var requestTaskSettings = new CustomElandRerankTaskSettings(Boolean.TRUE);
@@ -90,16 +97,6 @@ public void testOf_PrefersNonNullRequestTaskSettings() {
         assertThat(taskSettings, sameInstance(requestTaskSettings));
     }
 
-    public void testOf_UseOriginalSettings_IfRequestSettingsValuesAreNull() {
-        Boolean bool = null;
-        var originalSettings = new CustomElandRerankTaskSettings(Boolean.TRUE);
-        var requestTaskSettings = new CustomElandRerankTaskSettings(bool);
-
-        var taskSettings = CustomElandRerankTaskSettings.of(originalSettings, requestTaskSettings);
-
-        assertThat(taskSettings, sameInstance(originalSettings));
-    }
-
     private static CustomElandRerankTaskSettings createRandom() {
         return new CustomElandRerankTaskSettings(randomOptionalBoolean());
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java
index 419db748d793d..0db0a7669c8aa 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java
@@ -22,11 +22,18 @@ public class ElasticsearchInternalServiceSettingsTests extends AbstractWireSeria
 
     public static ElasticsearchInternalServiceSettings validInstance(String modelId) {
         boolean useAdaptive = randomBoolean();
+        var deploymentId = randomBoolean() ? null : randomAlphaOfLength(5);
         if (useAdaptive) {
             var adaptive = new AdaptiveAllocationsSettings(true, 1, randomIntBetween(2, 8));
-            return new ElasticsearchInternalServiceSettings(randomBoolean() ? 1 : null, randomIntBetween(1, 16), modelId, adaptive);
+            return new ElasticsearchInternalServiceSettings(
+                randomBoolean() ? 1 : null,
+                randomIntBetween(1, 16),
+                modelId,
+                adaptive,
+                deploymentId
+            );
         } else {
-            return new ElasticsearchInternalServiceSettings(randomIntBetween(1, 10), randomIntBetween(1, 16), modelId, null);
+            return new ElasticsearchInternalServiceSettings(randomIntBetween(1, 10), randomIntBetween(1, 16), modelId, null, deploymentId);
         }
     }
 
@@ -48,7 +55,8 @@ protected ElasticsearchInternalServiceSettings mutateInstance(ElasticsearchInter
                     instance.getNumAllocations() == null ? 1 : instance.getNumAllocations() + 1,
                     instance.getNumThreads(),
                     instance.modelId(),
-                    instance.getAdaptiveAllocationsSettings()
+                    instance.getAdaptiveAllocationsSettings(),
+                    instance.getDeploymentId()
                 )
             );
             case 1 -> new ElserInternalServiceSettings(
@@ -56,7 +64,8 @@ protected ElasticsearchInternalServiceSettings mutateInstance(ElasticsearchInter
                     instance.getNumAllocations(),
                     instance.getNumThreads() + 1,
                     instance.modelId(),
-                    instance.getAdaptiveAllocationsSettings()
+                    instance.getAdaptiveAllocationsSettings(),
+                    instance.getDeploymentId()
                 )
             );
             case 2 -> new ElserInternalServiceSettings(
@@ -64,7 +73,8 @@ protected ElasticsearchInternalServiceSettings mutateInstance(ElasticsearchInter
                     instance.getNumAllocations(),
                     instance.getNumThreads(),
                     instance.modelId() + "-bar",
-                    instance.getAdaptiveAllocationsSettings()
+                    instance.getAdaptiveAllocationsSettings(),
+                    instance.getDeploymentId()
                 )
             );
             default -> throw new IllegalStateException();
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
index 61645613b8722..5ec66687752a8 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
@@ -14,7 +14,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
@@ -38,6 +40,7 @@
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
@@ -49,7 +52,10 @@
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResultsTests;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResultsTests;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfig;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate;
 import org.elasticsearch.xpack.inference.InferencePlugin;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests;
@@ -171,7 +177,7 @@ public void testParseRequestConfig_Misconfigured() {
 
     public void testParseRequestConfig_E5() {
         {
-            var service = createService(mock(Client.class), Set.of("Aarch64"));
+            var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC);
             var settings = new HashMap<String, Object>();
             settings.put(
                 ModelConfigurations.SERVICE_SETTINGS,
@@ -198,7 +204,7 @@ public void testParseRequestConfig_E5() {
         }
 
         {
-            var service = createService(mock(Client.class), Set.of("linux-x86_64"));
+            var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.LINUX_X86_OPTIMIZED);
             var settings = new HashMap<String, Object>();
             settings.put(
                 ModelConfigurations.SERVICE_SETTINGS,
@@ -231,7 +237,7 @@ public void testParseRequestConfig_E5() {
 
         // Invalid service settings
         {
-            var service = createService(mock(Client.class), Set.of("Aarch64"));
+            var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC);
             var settings = new HashMap<String, Object>();
             settings.put(
                 ModelConfigurations.SERVICE_SETTINGS,
@@ -267,7 +273,7 @@ public void testParseRequestConfig_E5() {
                 }
             );
 
-            var service = createService(mock(Client.class), Set.of("Aarch64"));
+            var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC);
             var settings = new HashMap<String, Object>();
             settings.put(
                 ModelConfigurations.SERVICE_SETTINGS,
@@ -289,7 +295,7 @@ public void testParseRequestConfig_E5() {
 
         {
             assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled());
-            var service = createService(mock(Client.class), Set.of("Aarch64"));
+            var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC);
             var settings = new HashMap<String, Object>();
             settings.put(
                 ModelConfigurations.SERVICE_SETTINGS,
@@ -318,7 +324,7 @@ public void testParseRequestConfig_E5() {
 
         {
             assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled());
-            var service = createService(mock(Client.class), Set.of("Aarch64"));
+            var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC);
             var settings = new HashMap<String, Object>();
             settings.put(
                 ModelConfigurations.SERVICE_SETTINGS,
@@ -561,9 +567,9 @@ public void testParseRequestConfig_Rerank() {
             var client = mock(Client.class);
             doAnswer(invocation -> {
                 var listener = (ActionListener<GetTrainedModelsAction.Response>) invocation.getArguments()[2];
-                listener.onResponse(
-                    new GetTrainedModelsAction.Response(new QueryPage<>(List.of(mock(TrainedModelConfig.class)), 1, mock(ParseField.class)))
-                );
+                var modelConfig = mock(TrainedModelConfig.class);
+                when(modelConfig.getInferenceConfig()).thenReturn(mock(TextSimilarityConfig.class));
+                listener.onResponse(new GetTrainedModelsAction.Response(new QueryPage<>(List.of(modelConfig), 1, mock(ParseField.class))));
                 return null;
             }).when(client).execute(Mockito.same(GetTrainedModelsAction.INSTANCE), any(), any());
 
@@ -608,9 +614,9 @@ public void testParseRequestConfig_Rerank_DefaultTaskSettings() {
             var client = mock(Client.class);
             doAnswer(invocation -> {
                 var listener = (ActionListener<GetTrainedModelsAction.Response>) invocation.getArguments()[2];
-                listener.onResponse(
-                    new GetTrainedModelsAction.Response(new QueryPage<>(List.of(mock(TrainedModelConfig.class)), 1, mock(ParseField.class)))
-                );
+                var modelConfig = mock(TrainedModelConfig.class);
+                when(modelConfig.getInferenceConfig()).thenReturn(mock(TextSimilarityConfig.class));
+                listener.onResponse(new GetTrainedModelsAction.Response(new QueryPage<>(List.of(modelConfig), 1, mock(ParseField.class))));
                 return null;
             }).when(client).execute(Mockito.same(GetTrainedModelsAction.INSTANCE), any(), any());
 
@@ -707,9 +713,9 @@ private void testParseRequestConfig_SparseEmbedding(
         var client = mock(Client.class);
         doAnswer(invocation -> {
             var listener = (ActionListener<GetTrainedModelsAction.Response>) invocation.getArguments()[2];
-            listener.onResponse(
-                new GetTrainedModelsAction.Response(new QueryPage<>(List.of(mock(TrainedModelConfig.class)), 1, mock(ParseField.class)))
-            );
+            var modelConfig = mock(TrainedModelConfig.class);
+            when(modelConfig.getInferenceConfig()).thenReturn(mock(TextExpansionConfig.class));
+            listener.onResponse(new GetTrainedModelsAction.Response(new QueryPage<>(List.of(modelConfig), 1, mock(ParseField.class))));
             return null;
         }).when(client).execute(Mockito.same(GetTrainedModelsAction.INSTANCE), any(), any());
 
@@ -1300,14 +1306,20 @@ public void testParsePersistedConfig_Rerank() {
     }
 
     public void testParseRequestConfigEland_PreservesTaskType() {
+        var taskType = randomFrom(EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING));
+        var modelConfig = mock(TrainedModelConfig.class);
+        switch (taskType) {
+            case RERANK -> when(modelConfig.getInferenceConfig()).thenReturn(mock(TextSimilarityConfig.class));
+            case SPARSE_EMBEDDING -> when(modelConfig.getInferenceConfig()).thenReturn(mock(TextExpansionConfig.class));
+            case TEXT_EMBEDDING -> when(modelConfig.getInferenceConfig()).thenReturn(mock(TextEmbeddingConfig.class));
+        }
+
         var client = mock(Client.class);
         doAnswer(invocationOnMock -> {
             @SuppressWarnings("unchecked")
             ActionListener<GetTrainedModelsAction.Response> listener = (ActionListener<GetTrainedModelsAction.Response>) invocationOnMock
                 .getArguments()[2];
-            listener.onResponse(
-                new GetTrainedModelsAction.Response(new QueryPage<>(List.of(mock(TrainedModelConfig.class)), 1, mock(ParseField.class)))
-            );
+            listener.onResponse(new GetTrainedModelsAction.Response(new QueryPage<>(List.of(modelConfig), 1, mock(ParseField.class))));
             return Void.TYPE;
         }).when(client).execute(eq(GetTrainedModelsAction.INSTANCE), any(), any());
         when(client.threadPool()).thenReturn(threadPool);
@@ -1328,7 +1340,6 @@ public void testParseRequestConfigEland_PreservesTaskType() {
             )
         );
 
-        var taskType = randomFrom(EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING));
         CustomElandModel expectedModel = getCustomElandModel(taskType);
 
         PlainActionFuture<Model> listener = new PlainActionFuture<>();
@@ -1492,26 +1503,33 @@ public void testParseRequestConfigEland_SetsDimensionsToOne() {
 
     public void testModelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic() {
         {
-            var architectures = Set.of("Aarch64");
             assertFalse(
-                ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86)
+                ElasticsearchInternalService.modelVariantValidForArchitecture(
+                    BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC,
+                    MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86
+                )
             );
 
-            assertTrue(ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID));
-        }
-        {
-            var architectures = Set.of("linux-x86_64");
             assertTrue(
-                ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86)
+                ElasticsearchInternalService.modelVariantValidForArchitecture(
+                    BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC,
+                    MULTILINGUAL_E5_SMALL_MODEL_ID
+                )
             );
-            assertTrue(ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID));
         }
         {
-            var architectures = Set.of("linux-x86_64", "Aarch64");
-            assertFalse(
-                ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86)
+            assertTrue(
+                ElasticsearchInternalService.modelVariantValidForArchitecture(
+                    BaseElasticsearchInternalService.PreferredModelVariant.LINUX_X86_OPTIMIZED,
+                    MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86
+                )
+            );
+            assertTrue(
+                ElasticsearchInternalService.modelVariantValidForArchitecture(
+                    BaseElasticsearchInternalService.PreferredModelVariant.LINUX_X86_OPTIMIZED,
+                    MULTILINGUAL_E5_SMALL_MODEL_ID
+                )
             );
-            assertTrue(ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID));
         }
     }
 
@@ -1541,13 +1559,28 @@ public void testEmbeddingTypeFromTaskTypeAndSettings() {
         assertThat(e.getMessage(), containsString("Chunking is not supported for task type [completion]"));
     }
 
+    public void testIsDefaultId() {
+        var service = createService(mock(Client.class));
+        assertTrue(service.isDefaultId(".elser-2-elasticsearch"));
+        assertTrue(service.isDefaultId(".multilingual-e5-small-elasticsearch"));
+        assertFalse(service.isDefaultId("foo"));
+    }
+
     private ElasticsearchInternalService createService(Client client) {
-        var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool);
+        var cs = mock(ClusterService.class);
+        var cSettings = new ClusterSettings(Settings.EMPTY, Set.of(MachineLearningField.MAX_LAZY_ML_NODES));
+        when(cs.getClusterSettings()).thenReturn(cSettings);
+        var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool, cs, Settings.EMPTY);
         return new ElasticsearchInternalService(context);
     }
 
-    private ElasticsearchInternalService createService(Client client, Set<String> architectures) {
-        var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool);
-        return new ElasticsearchInternalService(context, l -> l.onResponse(architectures));
+    private ElasticsearchInternalService createService(Client client, BaseElasticsearchInternalService.PreferredModelVariant modelVariant) {
+        var context = new InferenceServiceExtension.InferenceServiceFactoryContext(
+            client,
+            threadPool,
+            mock(ClusterService.class),
+            Settings.EMPTY
+        );
+        return new ElasticsearchInternalService(context, l -> l.onResponse(modelVariant));
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java
new file mode 100644
index 0000000000000..96cd42efa42f5
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elasticsearch;
+
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+
+public class ElserInternalModelTests extends ESTestCase {
+    public void testUpdateNumAllocation() {
+        var model = new ElserInternalModel(
+            "foo",
+            TaskType.SPARSE_EMBEDDING,
+            ElasticsearchInternalService.NAME,
+            new ElserInternalServiceSettings(null, 1, "elser", null),
+            new ElserMlNodeTaskSettings(),
+            null
+        );
+
+        model.updateNumAllocations(1);
+        assertEquals(1, model.getServiceSettings().getNumAllocations().intValue());
+
+        model.updateNumAllocations(null);
+        assertNull(model.getServiceSettings().getNumAllocations());
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
index f4c13db78c4bc..3d523d7cab498 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
@@ -14,11 +14,15 @@
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 public class GoogleAiStudioCompletionModelTests extends ESTestCase {
 
@@ -55,14 +59,17 @@ public static GoogleAiStudioCompletionModel createModel(String model, String api
     }
 
     public static GoogleAiStudioCompletionModel createModel(String model, String url, String apiKey) {
-        return new GoogleAiStudioCompletionModel(
-            "id",
-            TaskType.COMPLETION,
-            "service",
-            url,
-            new GoogleAiStudioCompletionServiceSettings(model, null),
-            EmptyTaskSettings.INSTANCE,
-            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+        var googleModel = spy(
+            new GoogleAiStudioCompletionModel(
+                "id",
+                TaskType.COMPLETION,
+                "service",
+                new GoogleAiStudioCompletionServiceSettings(model, null),
+                EmptyTaskSettings.INSTANCE,
+                new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+            )
         );
+        when(googleModel.uri(anyBoolean())).thenReturn(URI.create(url));
+        return googleModel;
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java
index 95d3522b863a9..90738d43aacb3 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java
@@ -29,6 +29,15 @@ public static GoogleVertexAiSecretSettings createRandom() {
         return new GoogleVertexAiSecretSettings(randomSecureStringOfLength(30));
     }
 
+    public void testNewSecretSettings() {
+        GoogleVertexAiSecretSettings initialSettings = createRandom();
+        GoogleVertexAiSecretSettings newSettings = createRandom();
+        GoogleVertexAiSecretSettings newGoogleVertexAiSecretSettings = (GoogleVertexAiSecretSettings) initialSettings.newSecretSettings(
+            Map.of(GoogleVertexAiSecretSettings.SERVICE_ACCOUNT_JSON, newSettings.serviceAccountJson.toString())
+        );
+        assertEquals(newSettings, newGoogleVertexAiSecretSettings);
+    }
+
     public void testFromMap_ReturnsNull_WhenMapIsNUll() {
         assertNull(GoogleVertexAiSecretSettings.fromMap(null));
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java
index ac7e9348b370b..5b87bbc3c42c8 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -31,6 +32,23 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.autoTruncate() != null) {
+            newSettingsMap.put(GoogleVertexAiEmbeddingsTaskSettings.AUTO_TRUNCATE, newSettings.autoTruncate());
+        }
+        GoogleVertexAiEmbeddingsTaskSettings updatedSettings = (GoogleVertexAiEmbeddingsTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        if (newSettings.autoTruncate() == null) {
+            assertEquals(initialSettings.autoTruncate(), updatedSettings.autoTruncate());
+        } else {
+            assertEquals(newSettings.autoTruncate(), updatedSettings.autoTruncate());
+        }
+    }
+
     public void testFromMap_AutoTruncateIsSet() {
         var autoTruncate = true;
         var taskSettingsMap = getTaskSettingsMap(autoTruncate);
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java
index 03f89b6a2c042..957defb54d846 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -33,6 +34,23 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.topN() != null) {
+            newSettingsMap.put(GoogleVertexAiRerankTaskSettings.TOP_N, newSettings.topN());
+        }
+        GoogleVertexAiRerankTaskSettings updatedSettings = (GoogleVertexAiRerankTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        if (newSettings.topN() == null) {
+            assertEquals(initialSettings.topN(), updatedSettings.topN());
+        } else {
+            assertEquals(newSettings.topN(), updatedSettings.topN());
+        }
+    }
+
     public void testFromMap_TopNIsSet() {
         var topN = 1;
         var taskSettingsMap = getTaskSettingsMap(topN);
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
index beba9b1a92477..cf1438b334478 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
@@ -1007,7 +1007,6 @@ public void testInfer_SendsRequest() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest() throws Exception {
         String responseJson = """
             data: {\
@@ -1057,7 +1056,6 @@ private InferenceServiceResults streamChatCompletion() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114385")
     public void testInfer_StreamRequest_ErrorResponse() throws Exception {
         String responseJson = """
             {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java
index 16d7e8f1db9be..9d1170bb23dbb 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -31,6 +32,23 @@ public void testIsEmpty() {
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandomWithUser();
+        var newSettings = createRandomWithUser();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.user() != null) {
+            newSettingsMap.put(OpenAiServiceFields.USER, newSettings.user());
+        }
+        OpenAiChatCompletionTaskSettings updatedSettings = (OpenAiChatCompletionTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        if (newSettings.user() == null) {
+            assertEquals(initialSettings.user(), updatedSettings.user());
+        } else {
+            assertEquals(newSettings.user(), updatedSettings.user());
+        }
+    }
+
     public void testFromMap_WithUser() {
         assertEquals(
             new OpenAiChatCompletionTaskSettings("user"),
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java
index a5ae2f0a3a44b..0512c36e64de5 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java
@@ -14,9 +14,11 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields;
+import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionTaskSettings;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -37,11 +39,28 @@ public static OpenAiEmbeddingsTaskSettings createRandom() {
     }
 
     public void testIsEmpty() {
-        var randomSettings = createRandom();
+        var randomSettings = new OpenAiChatCompletionTaskSettings(randomBoolean() ? null : "username");
         var stringRep = Strings.toString(randomSettings);
         assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
     }
 
+    public void testUpdatedTaskSettings() {
+        var initialSettings = createRandom();
+        var newSettings = createRandom();
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        if (newSettings.user() != null) {
+            newSettingsMap.put(OpenAiServiceFields.USER, newSettings.user());
+        }
+        OpenAiEmbeddingsTaskSettings updatedSettings = (OpenAiEmbeddingsTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        if (newSettings.user() == null) {
+            assertEquals(initialSettings.user(), updatedSettings.user());
+        } else {
+            assertEquals(newSettings.user(), updatedSettings.user());
+        }
+    }
+
     public void testFromMap_WithUser() {
         assertEquals(
             new OpenAiEmbeddingsTaskSettings("user"),
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java
index 212a867349e5c..118cf25a452a7 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java
@@ -26,6 +26,15 @@ public static DefaultSecretSettings createRandom() {
         return new DefaultSecretSettings(new SecureString(randomAlphaOfLength(15).toCharArray()));
     }
 
+    public void testNewSecretSettings() {
+        DefaultSecretSettings initialSettings = createRandom();
+        DefaultSecretSettings newSettings = createRandom();
+        DefaultSecretSettings finalSettings = (DefaultSecretSettings) initialSettings.newSecretSettings(
+            Map.of(DefaultSecretSettings.API_KEY, newSettings.apiKey().toString())
+        );
+        assertEquals(newSettings, finalSettings);
+    }
+
     public void testFromMap() {
         var apiKey = "abc";
         var serviceSettings = DefaultSecretSettings.fromMap(new HashMap<>(Map.of(DefaultSecretSettings.API_KEY, apiKey)));
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
index 3f907ae1de6cd..71fb1fd95989f 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
@@ -343,3 +343,31 @@ setup:
                           inference_id: sparse-inference-id
                         another_field:
                           type: keyword
+
+---
+"Cannot be in an object field with subobjects disabled":
+  - requires:
+      cluster_features: "semantic_text.in_object_field_fix"
+      reason: object field fix added in 8.16.0 & 8.15.4
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-subobjects-index
+        body:
+          mappings:
+            properties:
+              level_1:
+                type: object
+                properties:
+                  level_2:
+                    type: object
+                    subobjects: false
+                    properties:
+                      sparse_field:
+                        type: semantic_text
+                        inference_id: sparse-inference-id
+
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "semantic_text field [level_1.level_2.sparse_field] cannot be in an object field with
+                            subobjects disabled" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
index 1795d754d2a9c..b4ba527a5bf45 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
@@ -578,3 +578,45 @@ setup:
   - match: { _source.sparse_field.text: "inference test" }
   - exists: _source.sparse_field.inference.chunks.0.embeddings
   - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+
+---
+"Can be used inside an object field":
+  - requires:
+      cluster_features: "semantic_text.in_object_field_fix"
+      reason: object field fix added in 8.16.0 & 8.15.4
+
+  - do:
+      indices.create:
+        index: test-in-object-index
+        body:
+          mappings:
+            properties:
+              level_1:
+                properties:
+                  sparse_field:
+                    type: semantic_text
+                    inference_id: sparse-inference-id
+                  dense_field:
+                    type: semantic_text
+                    inference_id: dense-inference-id
+
+  - do:
+      index:
+        index: test-in-object-index
+        id: doc_1
+        body:
+          level_1:
+            sparse_field: "inference test"
+            dense_field: "another inference test"
+
+  - do:
+      get:
+        index: test-in-object-index
+        id: doc_1
+
+  - match: { _source.level_1.sparse_field.text: "inference test" }
+  - exists: _source.level_1.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.level_1.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.level_1.dense_field.text: "another inference test" }
+  - exists: _source.level_1.dense_field.inference.chunks.0.embeddings
+  - match: { _source.level_1.dense_field.inference.chunks.0.text: "another inference test" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
index 9a4d7f4416164..2980d42d22c3e 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
@@ -1,4 +1,8 @@
 setup:
+  - skip:
+      features:
+        - close_to
+        - contains
   - requires:
       cluster_features: "text_similarity_reranker_retriever_supported"
       reason: semantic reranking introduced in 8.15.0
@@ -206,8 +210,8 @@ setup:
           size: 10
           explain: true
 
-  - match: { hits.hits.0._id: "doc_2" }
-  - match: { hits.hits.1._id: "doc_1" }
+  - contains: { hits.hits: { _id: "doc_2" } }
+  - contains: { hits.hits: { _id: "doc_1" } }
 
   - close_to: { hits.hits.0._explanation.value: { value: 0.4, error: 0.000001 } }
   - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml
index d33f57f763db8..47d87583fffb2 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml
@@ -8,6 +8,8 @@ setup:
       indices.create:
         index: test-index
         body:
+          settings:
+            number_of_shards: 1
           mappings:
             properties:
               text:
diff --git a/x-pack/plugin/kql/build.gradle b/x-pack/plugin/kql/build.gradle
new file mode 100644
index 0000000000000..198099329c7c0
--- /dev/null
+++ b/x-pack/plugin/kql/build.gradle
@@ -0,0 +1,114 @@
+import org.elasticsearch.gradle.internal.info.BuildParams
+import static org.elasticsearch.gradle.util.PlatformUtils.normalize
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.publish'
+
+esplugin {
+  name 'x-pack-kql'
+  description 'Elasticsearch Expanded Pack Plugin - KQL query'
+  classname 'org.elasticsearch.xpack.kql.KqlPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+base {
+  archivesName = 'x-pack-kql'
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('core'))
+  compileOnly "org.antlr:antlr4-runtime:${versions.antlr4}"
+
+  testImplementation "org.antlr:antlr4-runtime:${versions.antlr4}"
+  testImplementation project(':test:framework')
+  testImplementation(testArtifact(project(xpackModule('core'))))
+}
+
+/****************************************************************
+ *  Enable QA/rest integration tests for snapshot builds only   *
+ *  TODO: Enable for all builds upon this feature release       *
+ ****************************************************************/
+if (BuildParams.isSnapshotBuild()) {
+  addQaCheckDependencies(project)
+}
+
+/**********************************
+ * KQL parser configuration       *
+ **********************************/
+configurations {
+  regenerate
+}
+
+dependencies {
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
+}
+
+String grammarPath = 'src/main/antlr'
+String outputPath = 'src/main/java/org/elasticsearch/xpack/kql/parser'
+
+pluginManager.withPlugin('com.diffplug.spotless') {
+  spotless {
+    java {
+      // for some reason "${outputPath}/KqlBaser*.java" does not match the same files...
+      targetExclude "src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase*.java"
+    }
+  }
+}
+tasks.named('checkstyleMain').configure {
+  exclude { normalize(it.file.toString()).contains("src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase") }
+}
+
+tasks.register("cleanGenerated", Delete) {
+  delete fileTree(grammarPath) {
+    include '*.tokens'
+  }
+  delete fileTree(outputPath) {
+    include 'KqlBase*.java'
+    include 'KqlBase*.interp'
+  }
+}
+
+tasks.register("regenParser", JavaExec) {
+  dependsOn "cleanGenerated"
+  mainClass = 'org.antlr.v4.Tool'
+  classpath = configurations.regenerate
+  systemProperty 'file.encoding', 'UTF-8'
+  systemProperty 'user.language', 'en'
+  systemProperty 'user.country', 'US'
+  systemProperty 'user.variant', ''
+  args '-Werror',
+    '-package', 'org.elasticsearch.xpack.kql.parser',
+    '-listener',
+    '-visitor',
+    '-o', outputPath,
+    "${file(grammarPath)}/KqlBase.g4"
+}
+
+tasks.register("regen") {
+  dependsOn "regenParser"
+  doLast {
+    // moves token files to grammar directory for use with IDE's
+    ant.move(file: "${outputPath}/KqlBase.tokens", toDir: grammarPath)
+    ant.move(file: "${outputPath}/KqlBaseLexer.tokens", toDir: grammarPath)
+    // make the generated classes package private
+    ant.replaceregexp(match: 'public ((interface|class) \\QKqlBase\\E\\w+)',
+      replace: '\\1',
+      encoding: 'UTF-8') {
+      fileset(dir: outputPath, includes: 'KqlBase*.java')
+    }
+    // nuke timestamps/filenames in generated files
+    ant.replaceregexp(match: '\\Q// Generated from \\E.*',
+      replace: '\\/\\/ ANTLR GENERATED CODE: DO NOT EDIT',
+      encoding: 'UTF-8') {
+      fileset(dir: outputPath, includes: 'KqlBase*.java')
+    }
+    // remove tabs in antlr generated files
+    ant.replaceregexp(match: '\t', flags: 'g', replace: '    ', encoding: 'UTF-8') {
+      fileset(dir: outputPath, includes: 'KqlBase*.java')
+    }
+    // fix line endings
+    ant.fixcrlf(srcdir: outputPath, eol: 'lf') {
+      patternset(includes: 'KqlBase*.java')
+    }
+  }
+}
diff --git a/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 b/x-pack/plugin/kql/src/main/antlr/KqlBase.g4
new file mode 100644
index 0000000000000..cffa2db9f959a
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/antlr/KqlBase.g4
@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+grammar KqlBase;
+
+
+@header {
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+}
+
+options {
+  caseInsensitive=true;
+}
+
+topLevelQuery
+    : query? EOF
+    ;
+
+query
+    : query (AND | OR) query          #booleanQuery
+    | NOT subQuery=simpleQuery        #notQuery
+    | simpleQuery                     #defaultQuery
+    ;
+
+simpleQuery
+    : nestedQuery
+    | expression
+    | parenthesizedQuery
+    ;
+
+expression
+    : fieldTermQuery
+    | fieldRangeQuery
+    ;
+
+nestedQuery
+    : fieldName COLON LEFT_CURLY_BRACKET query RIGHT_CURLY_BRACKET
+    ;
+
+parenthesizedQuery:
+   LEFT_PARENTHESIS query RIGHT_PARENTHESIS;
+
+fieldRangeQuery
+    : fieldName operator=OP_COMPARE rangeQueryValue
+    ;
+
+fieldTermQuery
+    : (fieldName COLON)? termQueryValue
+    ;
+
+fieldName
+    : wildcardExpression
+    | unquotedLiteralExpression
+    | quotedStringExpression
+    ;
+
+rangeQueryValue
+    : unquotedLiteralExpression
+    | quotedStringExpression
+    ;
+
+termQueryValue
+    : wildcardExpression
+    | quotedStringExpression
+    | termValue=unquotedLiteralExpression
+    | groupingTermExpression;
+
+groupingTermExpression
+    : LEFT_PARENTHESIS unquotedLiteralExpression RIGHT_PARENTHESIS
+    ;
+
+unquotedLiteralExpression
+    : UNQUOTED_LITERAL+
+    ;
+
+quotedStringExpression
+    : QUOTED_STRING
+    ;
+
+wildcardExpression
+    : WILDCARD
+;
+
+
+DEFAULT_SKIP: WHITESPACE -> skip;
+
+AND: 'and';
+OR: 'or';
+NOT: 'not';
+
+COLON: ':';
+OP_COMPARE: OP_LESS | OP_MORE | OP_LESS_EQ | OP_MORE_EQ;
+
+LEFT_PARENTHESIS: '(';
+RIGHT_PARENTHESIS: ')';
+LEFT_CURLY_BRACKET: '{';
+RIGHT_CURLY_BRACKET: '}';
+
+UNQUOTED_LITERAL: WILDCARD* UNQUOTED_LITERAL_CHAR+ WILDCARD*;
+
+QUOTED_STRING: '"'QUOTED_CHAR*'"';
+
+WILDCARD: WILDCARD_CHAR+;
+
+fragment WILDCARD_CHAR: '*';
+fragment OP_LESS: '<';
+fragment OP_LESS_EQ: '<=';
+fragment OP_MORE: '>';
+fragment OP_MORE_EQ: '>=';
+
+fragment UNQUOTED_LITERAL_CHAR
+    : ESCAPED_WHITESPACE
+    | ESCAPED_SPECIAL_CHAR
+    | ESCAPE_UNICODE_SEQUENCE
+    | '\\' (AND | OR | NOT)
+    | WILDCARD_CHAR UNQUOTED_LITERAL_CHAR
+    | NON_SPECIAL_CHAR
+    ;
+
+fragment QUOTED_CHAR
+    : ESCAPED_WHITESPACE
+    | ESCAPE_UNICODE_SEQUENCE
+    | ESCAPED_QUOTE
+    | ~["]
+    ;
+
+fragment WHITESPACE: [ \t\n\r\u3000];
+fragment ESCAPED_WHITESPACE: '\\r' | '\\t' | '\\n';
+fragment NON_SPECIAL_CHAR: ~[ \\():<>"*{}];
+fragment ESCAPED_SPECIAL_CHAR: '\\'[ \\():<>"*{}];
+
+fragment ESCAPED_QUOTE: '\\"';
+
+fragment ESCAPE_UNICODE_SEQUENCE: '\\' UNICODE_SEQUENCE;
+fragment UNICODE_SEQUENCE: 'u' HEX_DIGIT HEX_DIGIT HEX_DIGIT HEX_DIGIT;
+fragment HEX_DIGIT: [0-9a-f];
diff --git a/x-pack/plugin/kql/src/main/antlr/KqlBase.tokens b/x-pack/plugin/kql/src/main/antlr/KqlBase.tokens
new file mode 100644
index 0000000000000..268ae0613b9f0
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/antlr/KqlBase.tokens
@@ -0,0 +1,21 @@
+DEFAULT_SKIP=1
+AND=2
+OR=3
+NOT=4
+COLON=5
+OP_COMPARE=6
+LEFT_PARENTHESIS=7
+RIGHT_PARENTHESIS=8
+LEFT_CURLY_BRACKET=9
+RIGHT_CURLY_BRACKET=10
+UNQUOTED_LITERAL=11
+QUOTED_STRING=12
+WILDCARD=13
+'and'=2
+'or'=3
+'not'=4
+':'=5
+'('=7
+')'=8
+'{'=9
+'}'=10
diff --git a/x-pack/plugin/kql/src/main/antlr/KqlBaseLexer.tokens b/x-pack/plugin/kql/src/main/antlr/KqlBaseLexer.tokens
new file mode 100644
index 0000000000000..268ae0613b9f0
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/antlr/KqlBaseLexer.tokens
@@ -0,0 +1,21 @@
+DEFAULT_SKIP=1
+AND=2
+OR=3
+NOT=4
+COLON=5
+OP_COMPARE=6
+LEFT_PARENTHESIS=7
+RIGHT_PARENTHESIS=8
+LEFT_CURLY_BRACKET=9
+RIGHT_CURLY_BRACKET=10
+UNQUOTED_LITERAL=11
+QUOTED_STRING=12
+WILDCARD=13
+'and'=2
+'or'=3
+'not'=4
+':'=5
+'('=7
+')'=8
+'{'=9
+'}'=10
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java
new file mode 100644
index 0000000000000..4734924b23618
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.kql;
+
+import org.elasticsearch.plugins.ExtensiblePlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+
+public class KqlPlugin extends Plugin implements SearchPlugin, ExtensiblePlugin {
+
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java
new file mode 100644
index 0000000000000..4ee7bdc1c7f21
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.kql.parser;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.SearchExecutionContext;
+
+class KqlAstBuilder extends KqlBaseBaseVisitor<QueryBuilder> {
+    private final SearchExecutionContext searchExecutionContext;
+
+    KqlAstBuilder(SearchExecutionContext searchExecutionContext) {
+        this.searchExecutionContext = searchExecutionContext;
+    }
+
+    public QueryBuilder toQueryBuilder(ParserRuleContext ctx) {
+        if (ctx instanceof KqlBaseParser.TopLevelQueryContext topLeveQueryContext) {
+            return new MatchAllQueryBuilder();
+        }
+
+        throw new IllegalArgumentException("context should be of type TopLevelQueryContext");
+    }
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp
new file mode 100644
index 0000000000000..1954195b52363
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp
@@ -0,0 +1,52 @@
+token literal names:
+null
+null
+'and'
+'or'
+'not'
+':'
+null
+'('
+')'
+'{'
+'}'
+null
+null
+null
+
+token symbolic names:
+null
+DEFAULT_SKIP
+AND
+OR
+NOT
+COLON
+OP_COMPARE
+LEFT_PARENTHESIS
+RIGHT_PARENTHESIS
+LEFT_CURLY_BRACKET
+RIGHT_CURLY_BRACKET
+UNQUOTED_LITERAL
+QUOTED_STRING
+WILDCARD
+
+rule names:
+topLevelQuery
+query
+simpleQuery
+expression
+nestedQuery
+parenthesizedQuery
+fieldRangeQuery
+fieldTermQuery
+fieldName
+rangeQueryValue
+termQueryValue
+groupingTermExpression
+unquotedLiteralExpression
+quotedStringExpression
+wildcardExpression
+
+
+atn:
+[4, 1, 13, 108, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 1, 0, 3, 0, 32, 8, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 40, 8, 1, 1, 1, 1, 1, 1, 1, 5, 1, 45, 8, 1, 10, 1, 12, 1, 48, 9, 1, 1, 2, 1, 2, 1, 2, 3, 2, 53, 8, 2, 1, 3, 1, 3, 3, 3, 57, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 3, 7, 76, 8, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 3, 8, 83, 8, 8, 1, 9, 1, 9, 3, 9, 87, 8, 9, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 93, 8, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 4, 12, 100, 8, 12, 11, 12, 12, 12, 101, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 0, 1, 2, 15, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 0, 1, 1, 0, 2, 3, 106, 0, 31, 1, 0, 0, 0, 2, 39, 1, 0, 0, 0, 4, 52, 1, 0, 0, 0, 6, 56, 1, 0, 0, 0, 8, 58, 1, 0, 0, 0, 10, 64, 1, 0, 0, 0, 12, 68, 1, 0, 0, 0, 14, 75, 1, 0, 0, 0, 16, 82, 1, 0, 0, 0, 18, 86, 1, 0, 0, 0, 20, 92, 1, 0, 0, 0, 22, 94, 1, 0, 0, 0, 24, 99, 1, 0, 0, 0, 26, 103, 1, 0, 0, 0, 28, 105, 1, 0, 0, 0, 30, 32, 3, 2, 1, 0, 31, 30, 1, 0, 0, 0, 31, 32, 1, 0, 0, 0, 32, 33, 1, 0, 0, 0, 33, 34, 5, 0, 0, 1, 34, 1, 1, 0, 0, 0, 35, 36, 6, 1, -1, 0, 36, 37, 5, 4, 0, 0, 37, 40, 3, 4, 2, 0, 38, 40, 3, 4, 2, 0, 39, 35, 1, 0, 0, 0, 39, 38, 1, 0, 0, 0, 40, 46, 1, 0, 0, 0, 41, 42, 10, 3, 0, 0, 42, 43, 7, 0, 0, 0, 43, 45, 3, 2, 1, 4, 44, 41, 1, 0, 0, 0, 45, 48, 1, 0, 0, 0, 46, 44, 1, 0, 0, 0, 46, 47, 1, 0, 0, 0, 47, 3, 1, 0, 0, 0, 48, 46, 1, 0, 0, 0, 49, 53, 3, 8, 4, 0, 50, 53, 3, 6, 3, 0, 51, 53, 3, 10, 5, 0, 52, 49, 1, 0, 0, 0, 52, 50, 1, 0, 0, 0, 52, 51, 1, 0, 0, 0, 53, 5, 1, 0, 0, 0, 54, 57, 3, 14, 7, 0, 55, 57, 3, 12, 6, 0, 56, 54, 1, 0, 0, 0, 56, 55, 1, 0, 0, 0, 57, 7, 1, 0, 0, 0, 58, 59, 3, 16, 8, 0, 59, 60, 5, 5, 0, 0, 60, 61, 5, 9, 0, 0, 61, 62, 3, 2, 1, 0, 62, 63, 5, 10, 0, 0, 63, 9, 1, 0, 0, 0, 64, 65, 5, 7, 0, 0, 65, 66, 3, 2, 1, 0, 66, 67, 5, 8, 0, 0, 67, 11, 1, 0, 0, 0, 68, 69, 3, 16, 8, 0, 69, 70, 5, 6, 0, 0, 70, 71, 3, 18, 9, 0, 71, 13, 1, 0, 0, 0, 72, 73, 3, 16, 8, 0, 73, 74, 5, 5, 0, 0, 74, 76, 1, 0, 0, 0, 75, 72, 1, 0, 0, 0, 75, 76, 1, 0, 0, 0, 76, 77, 1, 0, 0, 0, 77, 78, 3, 20, 10, 0, 78, 15, 1, 0, 0, 0, 79, 83, 3, 28, 14, 0, 80, 83, 3, 24, 12, 0, 81, 83, 3, 26, 13, 0, 82, 79, 1, 0, 0, 0, 82, 80, 1, 0, 0, 0, 82, 81, 1, 0, 0, 0, 83, 17, 1, 0, 0, 0, 84, 87, 3, 24, 12, 0, 85, 87, 3, 26, 13, 0, 86, 84, 1, 0, 0, 0, 86, 85, 1, 0, 0, 0, 87, 19, 1, 0, 0, 0, 88, 93, 3, 28, 14, 0, 89, 93, 3, 26, 13, 0, 90, 93, 3, 24, 12, 0, 91, 93, 3, 22, 11, 0, 92, 88, 1, 0, 0, 0, 92, 89, 1, 0, 0, 0, 92, 90, 1, 0, 0, 0, 92, 91, 1, 0, 0, 0, 93, 21, 1, 0, 0, 0, 94, 95, 5, 7, 0, 0, 95, 96, 3, 24, 12, 0, 96, 97, 5, 8, 0, 0, 97, 23, 1, 0, 0, 0, 98, 100, 5, 11, 0, 0, 99, 98, 1, 0, 0, 0, 100, 101, 1, 0, 0, 0, 101, 99, 1, 0, 0, 0, 101, 102, 1, 0, 0, 0, 102, 25, 1, 0, 0, 0, 103, 104, 5, 12, 0, 0, 104, 27, 1, 0, 0, 0, 105, 106, 5, 13, 0, 0, 106, 29, 1, 0, 0, 0, 10, 31, 39, 46, 52, 56, 75, 82, 86, 92, 101]
\ No newline at end of file
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseListener.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseListener.java
new file mode 100644
index 0000000000000..1b4282b5dbbea
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseListener.java
@@ -0,0 +1,252 @@
+// ANTLR GENERATED CODE: DO NOT EDIT
+package org.elasticsearch.xpack.kql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.tree.ErrorNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
+
+/**
+ * This class provides an empty implementation of {@link KqlBaseListener},
+ * which can be extended to create a listener which only needs to handle a subset
+ * of the available methods.
+ */
+@SuppressWarnings("CheckReturnValue")
+class KqlBaseBaseListener implements KqlBaseListener {
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterTopLevelQuery(KqlBaseParser.TopLevelQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitTopLevelQuery(KqlBaseParser.TopLevelQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterNotQuery(KqlBaseParser.NotQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitNotQuery(KqlBaseParser.NotQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterDefaultQuery(KqlBaseParser.DefaultQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitDefaultQuery(KqlBaseParser.DefaultQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterSimpleQuery(KqlBaseParser.SimpleQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitSimpleQuery(KqlBaseParser.SimpleQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterExpression(KqlBaseParser.ExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitExpression(KqlBaseParser.ExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterNestedQuery(KqlBaseParser.NestedQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitNestedQuery(KqlBaseParser.NestedQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterFieldRangeQuery(KqlBaseParser.FieldRangeQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitFieldRangeQuery(KqlBaseParser.FieldRangeQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterFieldTermQuery(KqlBaseParser.FieldTermQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitFieldTermQuery(KqlBaseParser.FieldTermQueryContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterFieldName(KqlBaseParser.FieldNameContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitFieldName(KqlBaseParser.FieldNameContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterRangeQueryValue(KqlBaseParser.RangeQueryValueContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitRangeQueryValue(KqlBaseParser.RangeQueryValueContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterTermQueryValue(KqlBaseParser.TermQueryValueContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitTermQueryValue(KqlBaseParser.TermQueryValueContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterGroupingTermExpression(KqlBaseParser.GroupingTermExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitGroupingTermExpression(KqlBaseParser.GroupingTermExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterUnquotedLiteralExpression(KqlBaseParser.UnquotedLiteralExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitUnquotedLiteralExpression(KqlBaseParser.UnquotedLiteralExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterQuotedStringExpression(KqlBaseParser.QuotedStringExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitQuotedStringExpression(KqlBaseParser.QuotedStringExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterWildcardExpression(KqlBaseParser.WildcardExpressionContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitWildcardExpression(KqlBaseParser.WildcardExpressionContext ctx) { }
+
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void enterEveryRule(ParserRuleContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void exitEveryRule(ParserRuleContext ctx) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void visitTerminal(TerminalNode node) { }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation does nothing.</p>
+     */
+    @Override public void visitErrorNode(ErrorNode node) { }
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseVisitor.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseVisitor.java
new file mode 100644
index 0000000000000..09cd668804154
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseBaseVisitor.java
@@ -0,0 +1,142 @@
+// ANTLR GENERATED CODE: DO NOT EDIT
+package org.elasticsearch.xpack.kql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
+
+/**
+ * This class provides an empty implementation of {@link KqlBaseVisitor},
+ * which can be extended to create a visitor which only needs to handle a subset
+ * of the available methods.
+ *
+ * @param <T> The return type of the visit operation. Use {@link Void} for
+ * operations with no return type.
+ */
+@SuppressWarnings("CheckReturnValue")
+class KqlBaseBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements KqlBaseVisitor<T> {
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitTopLevelQuery(KqlBaseParser.TopLevelQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitNotQuery(KqlBaseParser.NotQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitDefaultQuery(KqlBaseParser.DefaultQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitSimpleQuery(KqlBaseParser.SimpleQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitExpression(KqlBaseParser.ExpressionContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitNestedQuery(KqlBaseParser.NestedQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitFieldRangeQuery(KqlBaseParser.FieldRangeQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitFieldTermQuery(KqlBaseParser.FieldTermQueryContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitFieldName(KqlBaseParser.FieldNameContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitRangeQueryValue(KqlBaseParser.RangeQueryValueContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitTermQueryValue(KqlBaseParser.TermQueryValueContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitGroupingTermExpression(KqlBaseParser.GroupingTermExpressionContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitUnquotedLiteralExpression(KqlBaseParser.UnquotedLiteralExpressionContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitQuotedStringExpression(KqlBaseParser.QuotedStringExpressionContext ctx) { return visitChildren(ctx); }
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default implementation returns the result of calling
+     * {@link #visitChildren} on {@code ctx}.</p>
+     */
+    @Override public T visitWildcardExpression(KqlBaseParser.WildcardExpressionContext ctx) { return visitChildren(ctx); }
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseLexer.interp b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseLexer.interp
new file mode 100644
index 0000000000000..d178df5fcbc88
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseLexer.interp
@@ -0,0 +1,71 @@
+token literal names:
+null
+null
+'and'
+'or'
+'not'
+':'
+null
+'('
+')'
+'{'
+'}'
+null
+null
+null
+
+token symbolic names:
+null
+DEFAULT_SKIP
+AND
+OR
+NOT
+COLON
+OP_COMPARE
+LEFT_PARENTHESIS
+RIGHT_PARENTHESIS
+LEFT_CURLY_BRACKET
+RIGHT_CURLY_BRACKET
+UNQUOTED_LITERAL
+QUOTED_STRING
+WILDCARD
+
+rule names:
+DEFAULT_SKIP
+AND
+OR
+NOT
+COLON
+OP_COMPARE
+LEFT_PARENTHESIS
+RIGHT_PARENTHESIS
+LEFT_CURLY_BRACKET
+RIGHT_CURLY_BRACKET
+UNQUOTED_LITERAL
+QUOTED_STRING
+WILDCARD
+WILDCARD_CHAR
+OP_LESS
+OP_LESS_EQ
+OP_MORE
+OP_MORE_EQ
+UNQUOTED_LITERAL_CHAR
+QUOTED_CHAR
+WHITESPACE
+ESCAPED_WHITESPACE
+NON_SPECIAL_CHAR
+ESCAPED_SPECIAL_CHAR
+ESCAPED_QUOTE
+ESCAPE_UNICODE_SEQUENCE
+UNICODE_SEQUENCE
+HEX_DIGIT
+
+channel names:
+DEFAULT_TOKEN_CHANNEL
+HIDDEN
+
+mode names:
+DEFAULT_MODE
+
+atn:
+[4, 0, 13, 181, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 79, 8, 5, 1, 6, 1, 6, 1, 7, 1, 7, 1, 8, 1, 8, 1, 9, 1, 9, 1, 10, 5, 10, 90, 8, 10, 10, 10, 12, 10, 93, 9, 10, 1, 10, 4, 10, 96, 8, 10, 11, 10, 12, 10, 97, 1, 10, 5, 10, 101, 8, 10, 10, 10, 12, 10, 104, 9, 10, 1, 11, 1, 11, 5, 11, 108, 8, 11, 10, 11, 12, 11, 111, 9, 11, 1, 11, 1, 11, 1, 12, 4, 12, 116, 8, 12, 11, 12, 12, 12, 117, 1, 13, 1, 13, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 139, 8, 18, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 145, 8, 18, 1, 19, 1, 19, 1, 19, 1, 19, 3, 19, 151, 8, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 3, 21, 161, 8, 21, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 0, 0, 28, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25, 13, 27, 0, 29, 0, 31, 0, 33, 0, 35, 0, 37, 0, 39, 0, 41, 0, 43, 0, 45, 0, 47, 0, 49, 0, 51, 0, 53, 0, 55, 0, 1, 0, 11, 2, 0, 65, 65, 97, 97, 2, 0, 78, 78, 110, 110, 2, 0, 68, 68, 100, 100, 2, 0, 79, 79, 111, 111, 2, 0, 82, 82, 114, 114, 2, 0, 84, 84, 116, 116, 1, 0, 34, 34, 4, 0, 9, 10, 13, 13, 32, 32, 12288, 12288, 9, 0, 32, 32, 34, 34, 40, 42, 58, 58, 60, 60, 62, 62, 92, 92, 123, 123, 125, 125, 2, 0, 85, 85, 117, 117, 3, 0, 48, 57, 65, 70, 97, 102, 185, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 1, 57, 1, 0, 0, 0, 3, 61, 1, 0, 0, 0, 5, 65, 1, 0, 0, 0, 7, 68, 1, 0, 0, 0, 9, 72, 1, 0, 0, 0, 11, 78, 1, 0, 0, 0, 13, 80, 1, 0, 0, 0, 15, 82, 1, 0, 0, 0, 17, 84, 1, 0, 0, 0, 19, 86, 1, 0, 0, 0, 21, 91, 1, 0, 0, 0, 23, 105, 1, 0, 0, 0, 25, 115, 1, 0, 0, 0, 27, 119, 1, 0, 0, 0, 29, 121, 1, 0, 0, 0, 31, 123, 1, 0, 0, 0, 33, 126, 1, 0, 0, 0, 35, 128, 1, 0, 0, 0, 37, 144, 1, 0, 0, 0, 39, 150, 1, 0, 0, 0, 41, 152, 1, 0, 0, 0, 43, 160, 1, 0, 0, 0, 45, 162, 1, 0, 0, 0, 47, 164, 1, 0, 0, 0, 49, 167, 1, 0, 0, 0, 51, 170, 1, 0, 0, 0, 53, 173, 1, 0, 0, 0, 55, 179, 1, 0, 0, 0, 57, 58, 3, 41, 20, 0, 58, 59, 1, 0, 0, 0, 59, 60, 6, 0, 0, 0, 60, 2, 1, 0, 0, 0, 61, 62, 7, 0, 0, 0, 62, 63, 7, 1, 0, 0, 63, 64, 7, 2, 0, 0, 64, 4, 1, 0, 0, 0, 65, 66, 7, 3, 0, 0, 66, 67, 7, 4, 0, 0, 67, 6, 1, 0, 0, 0, 68, 69, 7, 1, 0, 0, 69, 70, 7, 3, 0, 0, 70, 71, 7, 5, 0, 0, 71, 8, 1, 0, 0, 0, 72, 73, 5, 58, 0, 0, 73, 10, 1, 0, 0, 0, 74, 79, 3, 29, 14, 0, 75, 79, 3, 33, 16, 0, 76, 79, 3, 31, 15, 0, 77, 79, 3, 35, 17, 0, 78, 74, 1, 0, 0, 0, 78, 75, 1, 0, 0, 0, 78, 76, 1, 0, 0, 0, 78, 77, 1, 0, 0, 0, 79, 12, 1, 0, 0, 0, 80, 81, 5, 40, 0, 0, 81, 14, 1, 0, 0, 0, 82, 83, 5, 41, 0, 0, 83, 16, 1, 0, 0, 0, 84, 85, 5, 123, 0, 0, 85, 18, 1, 0, 0, 0, 86, 87, 5, 125, 0, 0, 87, 20, 1, 0, 0, 0, 88, 90, 3, 25, 12, 0, 89, 88, 1, 0, 0, 0, 90, 93, 1, 0, 0, 0, 91, 89, 1, 0, 0, 0, 91, 92, 1, 0, 0, 0, 92, 95, 1, 0, 0, 0, 93, 91, 1, 0, 0, 0, 94, 96, 3, 37, 18, 0, 95, 94, 1, 0, 0, 0, 96, 97, 1, 0, 0, 0, 97, 95, 1, 0, 0, 0, 97, 98, 1, 0, 0, 0, 98, 102, 1, 0, 0, 0, 99, 101, 3, 25, 12, 0, 100, 99, 1, 0, 0, 0, 101, 104, 1, 0, 0, 0, 102, 100, 1, 0, 0, 0, 102, 103, 1, 0, 0, 0, 103, 22, 1, 0, 0, 0, 104, 102, 1, 0, 0, 0, 105, 109, 5, 34, 0, 0, 106, 108, 3, 39, 19, 0, 107, 106, 1, 0, 0, 0, 108, 111, 1, 0, 0, 0, 109, 107, 1, 0, 0, 0, 109, 110, 1, 0, 0, 0, 110, 112, 1, 0, 0, 0, 111, 109, 1, 0, 0, 0, 112, 113, 5, 34, 0, 0, 113, 24, 1, 0, 0, 0, 114, 116, 3, 27, 13, 0, 115, 114, 1, 0, 0, 0, 116, 117, 1, 0, 0, 0, 117, 115, 1, 0, 0, 0, 117, 118, 1, 0, 0, 0, 118, 26, 1, 0, 0, 0, 119, 120, 5, 42, 0, 0, 120, 28, 1, 0, 0, 0, 121, 122, 5, 60, 0, 0, 122, 30, 1, 0, 0, 0, 123, 124, 5, 60, 0, 0, 124, 125, 5, 61, 0, 0, 125, 32, 1, 0, 0, 0, 126, 127, 5, 62, 0, 0, 127, 34, 1, 0, 0, 0, 128, 129, 5, 62, 0, 0, 129, 130, 5, 61, 0, 0, 130, 36, 1, 0, 0, 0, 131, 145, 3, 43, 21, 0, 132, 145, 3, 47, 23, 0, 133, 145, 3, 51, 25, 0, 134, 138, 5, 92, 0, 0, 135, 139, 3, 3, 1, 0, 136, 139, 3, 5, 2, 0, 137, 139, 3, 7, 3, 0, 138, 135, 1, 0, 0, 0, 138, 136, 1, 0, 0, 0, 138, 137, 1, 0, 0, 0, 139, 145, 1, 0, 0, 0, 140, 141, 3, 27, 13, 0, 141, 142, 3, 37, 18, 0, 142, 145, 1, 0, 0, 0, 143, 145, 3, 45, 22, 0, 144, 131, 1, 0, 0, 0, 144, 132, 1, 0, 0, 0, 144, 133, 1, 0, 0, 0, 144, 134, 1, 0, 0, 0, 144, 140, 1, 0, 0, 0, 144, 143, 1, 0, 0, 0, 145, 38, 1, 0, 0, 0, 146, 151, 3, 43, 21, 0, 147, 151, 3, 51, 25, 0, 148, 151, 3, 49, 24, 0, 149, 151, 8, 6, 0, 0, 150, 146, 1, 0, 0, 0, 150, 147, 1, 0, 0, 0, 150, 148, 1, 0, 0, 0, 150, 149, 1, 0, 0, 0, 151, 40, 1, 0, 0, 0, 152, 153, 7, 7, 0, 0, 153, 42, 1, 0, 0, 0, 154, 155, 5, 92, 0, 0, 155, 161, 7, 4, 0, 0, 156, 157, 5, 92, 0, 0, 157, 161, 7, 5, 0, 0, 158, 159, 5, 92, 0, 0, 159, 161, 7, 1, 0, 0, 160, 154, 1, 0, 0, 0, 160, 156, 1, 0, 0, 0, 160, 158, 1, 0, 0, 0, 161, 44, 1, 0, 0, 0, 162, 163, 8, 8, 0, 0, 163, 46, 1, 0, 0, 0, 164, 165, 5, 92, 0, 0, 165, 166, 7, 8, 0, 0, 166, 48, 1, 0, 0, 0, 167, 168, 5, 92, 0, 0, 168, 169, 5, 34, 0, 0, 169, 50, 1, 0, 0, 0, 170, 171, 5, 92, 0, 0, 171, 172, 3, 53, 26, 0, 172, 52, 1, 0, 0, 0, 173, 174, 7, 9, 0, 0, 174, 175, 3, 55, 27, 0, 175, 176, 3, 55, 27, 0, 176, 177, 3, 55, 27, 0, 177, 178, 3, 55, 27, 0, 178, 54, 1, 0, 0, 0, 179, 180, 7, 10, 0, 0, 180, 56, 1, 0, 0, 0, 11, 0, 78, 91, 97, 102, 109, 117, 138, 144, 150, 160, 1, 6, 0, 0]
\ No newline at end of file
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseLexer.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseLexer.java
new file mode 100644
index 0000000000000..b397a412d5e8e
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseLexer.java
@@ -0,0 +1,245 @@
+// ANTLR GENERATED CODE: DO NOT EDIT
+package org.elasticsearch.xpack.kql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.*;
+import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.dfa.DFA;
+import org.antlr.v4.runtime.misc.*;
+
+@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue", "this-escape"})
+class KqlBaseLexer extends Lexer {
+    static { RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); }
+
+    protected static final DFA[] _decisionToDFA;
+    protected static final PredictionContextCache _sharedContextCache =
+        new PredictionContextCache();
+    public static final int
+        DEFAULT_SKIP=1, AND=2, OR=3, NOT=4, COLON=5, OP_COMPARE=6, LEFT_PARENTHESIS=7, 
+        RIGHT_PARENTHESIS=8, LEFT_CURLY_BRACKET=9, RIGHT_CURLY_BRACKET=10, UNQUOTED_LITERAL=11, 
+        QUOTED_STRING=12, WILDCARD=13;
+    public static String[] channelNames = {
+        "DEFAULT_TOKEN_CHANNEL", "HIDDEN"
+    };
+
+    public static String[] modeNames = {
+        "DEFAULT_MODE"
+    };
+
+    private static String[] makeRuleNames() {
+        return new String[] {
+            "DEFAULT_SKIP", "AND", "OR", "NOT", "COLON", "OP_COMPARE", "LEFT_PARENTHESIS", 
+            "RIGHT_PARENTHESIS", "LEFT_CURLY_BRACKET", "RIGHT_CURLY_BRACKET", "UNQUOTED_LITERAL", 
+            "QUOTED_STRING", "WILDCARD", "WILDCARD_CHAR", "OP_LESS", "OP_LESS_EQ", 
+            "OP_MORE", "OP_MORE_EQ", "UNQUOTED_LITERAL_CHAR", "QUOTED_CHAR", "WHITESPACE", 
+            "ESCAPED_WHITESPACE", "NON_SPECIAL_CHAR", "ESCAPED_SPECIAL_CHAR", "ESCAPED_QUOTE", 
+            "ESCAPE_UNICODE_SEQUENCE", "UNICODE_SEQUENCE", "HEX_DIGIT"
+        };
+    }
+    public static final String[] ruleNames = makeRuleNames();
+
+    private static String[] makeLiteralNames() {
+        return new String[] {
+            null, null, "'and'", "'or'", "'not'", "':'", null, "'('", "')'", "'{'", 
+            "'}'"
+        };
+    }
+    private static final String[] _LITERAL_NAMES = makeLiteralNames();
+    private static String[] makeSymbolicNames() {
+        return new String[] {
+            null, "DEFAULT_SKIP", "AND", "OR", "NOT", "COLON", "OP_COMPARE", "LEFT_PARENTHESIS", 
+            "RIGHT_PARENTHESIS", "LEFT_CURLY_BRACKET", "RIGHT_CURLY_BRACKET", "UNQUOTED_LITERAL", 
+            "QUOTED_STRING", "WILDCARD"
+        };
+    }
+    private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
+    public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
+
+    /**
+     * @deprecated Use {@link #VOCABULARY} instead.
+     */
+    @Deprecated
+    public static final String[] tokenNames;
+    static {
+        tokenNames = new String[_SYMBOLIC_NAMES.length];
+        for (int i = 0; i < tokenNames.length; i++) {
+            tokenNames[i] = VOCABULARY.getLiteralName(i);
+            if (tokenNames[i] == null) {
+                tokenNames[i] = VOCABULARY.getSymbolicName(i);
+            }
+
+            if (tokenNames[i] == null) {
+                tokenNames[i] = "<INVALID>";
+            }
+        }
+    }
+
+    @Override
+    @Deprecated
+    public String[] getTokenNames() {
+        return tokenNames;
+    }
+
+    @Override
+
+    public Vocabulary getVocabulary() {
+        return VOCABULARY;
+    }
+
+
+    public KqlBaseLexer(CharStream input) {
+        super(input);
+        _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
+    }
+
+    @Override
+    public String getGrammarFileName() { return "KqlBase.g4"; }
+
+    @Override
+    public String[] getRuleNames() { return ruleNames; }
+
+    @Override
+    public String getSerializedATN() { return _serializedATN; }
+
+    @Override
+    public String[] getChannelNames() { return channelNames; }
+
+    @Override
+    public String[] getModeNames() { return modeNames; }
+
+    @Override
+    public ATN getATN() { return _ATN; }
+
+    public static final String _serializedATN =
+        "\u0004\u0000\r\u00b5\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001"+
+        "\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004"+
+        "\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007"+
+        "\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b"+
+        "\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002"+
+        "\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002"+
+        "\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002"+
+        "\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002"+
+        "\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002"+
+        "\u001b\u0007\u001b\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+
+        "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001"+
+        "\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001"+
+        "\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005O\b"+
+        "\u0005\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007\u0001\b\u0001\b"+
+        "\u0001\t\u0001\t\u0001\n\u0005\nZ\b\n\n\n\f\n]\t\n\u0001\n\u0004\n`\b"+
+        "\n\u000b\n\f\na\u0001\n\u0005\ne\b\n\n\n\f\nh\t\n\u0001\u000b\u0001\u000b"+
+        "\u0005\u000bl\b\u000b\n\u000b\f\u000bo\t\u000b\u0001\u000b\u0001\u000b"+
+        "\u0001\f\u0004\ft\b\f\u000b\f\f\fu\u0001\r\u0001\r\u0001\u000e\u0001\u000e"+
+        "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0011"+
+        "\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+
+        "\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u008b\b\u0012\u0001\u0012"+
+        "\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u0091\b\u0012\u0001\u0013"+
+        "\u0001\u0013\u0001\u0013\u0001\u0013\u0003\u0013\u0097\b\u0013\u0001\u0014"+
+        "\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015"+
+        "\u0001\u0015\u0003\u0015\u00a1\b\u0015\u0001\u0016\u0001\u0016\u0001\u0017"+
+        "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"+
+        "\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a"+
+        "\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0000\u0000\u001c\u0001"+
+        "\u0001\u0003\u0002\u0005\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007"+
+        "\u000f\b\u0011\t\u0013\n\u0015\u000b\u0017\f\u0019\r\u001b\u0000\u001d"+
+        "\u0000\u001f\u0000!\u0000#\u0000%\u0000\'\u0000)\u0000+\u0000-\u0000/"+
+        "\u00001\u00003\u00005\u00007\u0000\u0001\u0000\u000b\u0002\u0000AAaa\u0002"+
+        "\u0000NNnn\u0002\u0000DDdd\u0002\u0000OOoo\u0002\u0000RRrr\u0002\u0000"+
+        "TTtt\u0001\u0000\"\"\u0004\u0000\t\n\r\r  \u3000\u3000\t\u0000  \"\"("+
+        "*::<<>>\\\\{{}}\u0002\u0000UUuu\u0003\u000009AFaf\u00b9\u0000\u0001\u0001"+
+        "\u0000\u0000\u0000\u0000\u0003\u0001\u0000\u0000\u0000\u0000\u0005\u0001"+
+        "\u0000\u0000\u0000\u0000\u0007\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000"+
+        "\u0000\u0000\u0000\u000b\u0001\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000"+
+        "\u0000\u0000\u000f\u0001\u0000\u0000\u0000\u0000\u0011\u0001\u0000\u0000"+
+        "\u0000\u0000\u0013\u0001\u0000\u0000\u0000\u0000\u0015\u0001\u0000\u0000"+
+        "\u0000\u0000\u0017\u0001\u0000\u0000\u0000\u0000\u0019\u0001\u0000\u0000"+
+        "\u0000\u00019\u0001\u0000\u0000\u0000\u0003=\u0001\u0000\u0000\u0000\u0005"+
+        "A\u0001\u0000\u0000\u0000\u0007D\u0001\u0000\u0000\u0000\tH\u0001\u0000"+
+        "\u0000\u0000\u000bN\u0001\u0000\u0000\u0000\rP\u0001\u0000\u0000\u0000"+
+        "\u000fR\u0001\u0000\u0000\u0000\u0011T\u0001\u0000\u0000\u0000\u0013V"+
+        "\u0001\u0000\u0000\u0000\u0015[\u0001\u0000\u0000\u0000\u0017i\u0001\u0000"+
+        "\u0000\u0000\u0019s\u0001\u0000\u0000\u0000\u001bw\u0001\u0000\u0000\u0000"+
+        "\u001dy\u0001\u0000\u0000\u0000\u001f{\u0001\u0000\u0000\u0000!~\u0001"+
+        "\u0000\u0000\u0000#\u0080\u0001\u0000\u0000\u0000%\u0090\u0001\u0000\u0000"+
+        "\u0000\'\u0096\u0001\u0000\u0000\u0000)\u0098\u0001\u0000\u0000\u0000"+
+        "+\u00a0\u0001\u0000\u0000\u0000-\u00a2\u0001\u0000\u0000\u0000/\u00a4"+
+        "\u0001\u0000\u0000\u00001\u00a7\u0001\u0000\u0000\u00003\u00aa\u0001\u0000"+
+        "\u0000\u00005\u00ad\u0001\u0000\u0000\u00007\u00b3\u0001\u0000\u0000\u0000"+
+        "9:\u0003)\u0014\u0000:;\u0001\u0000\u0000\u0000;<\u0006\u0000\u0000\u0000"+
+        "<\u0002\u0001\u0000\u0000\u0000=>\u0007\u0000\u0000\u0000>?\u0007\u0001"+
+        "\u0000\u0000?@\u0007\u0002\u0000\u0000@\u0004\u0001\u0000\u0000\u0000"+
+        "AB\u0007\u0003\u0000\u0000BC\u0007\u0004\u0000\u0000C\u0006\u0001\u0000"+
+        "\u0000\u0000DE\u0007\u0001\u0000\u0000EF\u0007\u0003\u0000\u0000FG\u0007"+
+        "\u0005\u0000\u0000G\b\u0001\u0000\u0000\u0000HI\u0005:\u0000\u0000I\n"+
+        "\u0001\u0000\u0000\u0000JO\u0003\u001d\u000e\u0000KO\u0003!\u0010\u0000"+
+        "LO\u0003\u001f\u000f\u0000MO\u0003#\u0011\u0000NJ\u0001\u0000\u0000\u0000"+
+        "NK\u0001\u0000\u0000\u0000NL\u0001\u0000\u0000\u0000NM\u0001\u0000\u0000"+
+        "\u0000O\f\u0001\u0000\u0000\u0000PQ\u0005(\u0000\u0000Q\u000e\u0001\u0000"+
+        "\u0000\u0000RS\u0005)\u0000\u0000S\u0010\u0001\u0000\u0000\u0000TU\u0005"+
+        "{\u0000\u0000U\u0012\u0001\u0000\u0000\u0000VW\u0005}\u0000\u0000W\u0014"+
+        "\u0001\u0000\u0000\u0000XZ\u0003\u0019\f\u0000YX\u0001\u0000\u0000\u0000"+
+        "Z]\u0001\u0000\u0000\u0000[Y\u0001\u0000\u0000\u0000[\\\u0001\u0000\u0000"+
+        "\u0000\\_\u0001\u0000\u0000\u0000][\u0001\u0000\u0000\u0000^`\u0003%\u0012"+
+        "\u0000_^\u0001\u0000\u0000\u0000`a\u0001\u0000\u0000\u0000a_\u0001\u0000"+
+        "\u0000\u0000ab\u0001\u0000\u0000\u0000bf\u0001\u0000\u0000\u0000ce\u0003"+
+        "\u0019\f\u0000dc\u0001\u0000\u0000\u0000eh\u0001\u0000\u0000\u0000fd\u0001"+
+        "\u0000\u0000\u0000fg\u0001\u0000\u0000\u0000g\u0016\u0001\u0000\u0000"+
+        "\u0000hf\u0001\u0000\u0000\u0000im\u0005\"\u0000\u0000jl\u0003\'\u0013"+
+        "\u0000kj\u0001\u0000\u0000\u0000lo\u0001\u0000\u0000\u0000mk\u0001\u0000"+
+        "\u0000\u0000mn\u0001\u0000\u0000\u0000np\u0001\u0000\u0000\u0000om\u0001"+
+        "\u0000\u0000\u0000pq\u0005\"\u0000\u0000q\u0018\u0001\u0000\u0000\u0000"+
+        "rt\u0003\u001b\r\u0000sr\u0001\u0000\u0000\u0000tu\u0001\u0000\u0000\u0000"+
+        "us\u0001\u0000\u0000\u0000uv\u0001\u0000\u0000\u0000v\u001a\u0001\u0000"+
+        "\u0000\u0000wx\u0005*\u0000\u0000x\u001c\u0001\u0000\u0000\u0000yz\u0005"+
+        "<\u0000\u0000z\u001e\u0001\u0000\u0000\u0000{|\u0005<\u0000\u0000|}\u0005"+
+        "=\u0000\u0000} \u0001\u0000\u0000\u0000~\u007f\u0005>\u0000\u0000\u007f"+
+        "\"\u0001\u0000\u0000\u0000\u0080\u0081\u0005>\u0000\u0000\u0081\u0082"+
+        "\u0005=\u0000\u0000\u0082$\u0001\u0000\u0000\u0000\u0083\u0091\u0003+"+
+        "\u0015\u0000\u0084\u0091\u0003/\u0017\u0000\u0085\u0091\u00033\u0019\u0000"+
+        "\u0086\u008a\u0005\\\u0000\u0000\u0087\u008b\u0003\u0003\u0001\u0000\u0088"+
+        "\u008b\u0003\u0005\u0002\u0000\u0089\u008b\u0003\u0007\u0003\u0000\u008a"+
+        "\u0087\u0001\u0000\u0000\u0000\u008a\u0088\u0001\u0000\u0000\u0000\u008a"+
+        "\u0089\u0001\u0000\u0000\u0000\u008b\u0091\u0001\u0000\u0000\u0000\u008c"+
+        "\u008d\u0003\u001b\r\u0000\u008d\u008e\u0003%\u0012\u0000\u008e\u0091"+
+        "\u0001\u0000\u0000\u0000\u008f\u0091\u0003-\u0016\u0000\u0090\u0083\u0001"+
+        "\u0000\u0000\u0000\u0090\u0084\u0001\u0000\u0000\u0000\u0090\u0085\u0001"+
+        "\u0000\u0000\u0000\u0090\u0086\u0001\u0000\u0000\u0000\u0090\u008c\u0001"+
+        "\u0000\u0000\u0000\u0090\u008f\u0001\u0000\u0000\u0000\u0091&\u0001\u0000"+
+        "\u0000\u0000\u0092\u0097\u0003+\u0015\u0000\u0093\u0097\u00033\u0019\u0000"+
+        "\u0094\u0097\u00031\u0018\u0000\u0095\u0097\b\u0006\u0000\u0000\u0096"+
+        "\u0092\u0001\u0000\u0000\u0000\u0096\u0093\u0001\u0000\u0000\u0000\u0096"+
+        "\u0094\u0001\u0000\u0000\u0000\u0096\u0095\u0001\u0000\u0000\u0000\u0097"+
+        "(\u0001\u0000\u0000\u0000\u0098\u0099\u0007\u0007\u0000\u0000\u0099*\u0001"+
+        "\u0000\u0000\u0000\u009a\u009b\u0005\\\u0000\u0000\u009b\u00a1\u0007\u0004"+
+        "\u0000\u0000\u009c\u009d\u0005\\\u0000\u0000\u009d\u00a1\u0007\u0005\u0000"+
+        "\u0000\u009e\u009f\u0005\\\u0000\u0000\u009f\u00a1\u0007\u0001\u0000\u0000"+
+        "\u00a0\u009a\u0001\u0000\u0000\u0000\u00a0\u009c\u0001\u0000\u0000\u0000"+
+        "\u00a0\u009e\u0001\u0000\u0000\u0000\u00a1,\u0001\u0000\u0000\u0000\u00a2"+
+        "\u00a3\b\b\u0000\u0000\u00a3.\u0001\u0000\u0000\u0000\u00a4\u00a5\u0005"+
+        "\\\u0000\u0000\u00a5\u00a6\u0007\b\u0000\u0000\u00a60\u0001\u0000\u0000"+
+        "\u0000\u00a7\u00a8\u0005\\\u0000\u0000\u00a8\u00a9\u0005\"\u0000\u0000"+
+        "\u00a92\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\\\u0000\u0000\u00ab"+
+        "\u00ac\u00035\u001a\u0000\u00ac4\u0001\u0000\u0000\u0000\u00ad\u00ae\u0007"+
+        "\t\u0000\u0000\u00ae\u00af\u00037\u001b\u0000\u00af\u00b0\u00037\u001b"+
+        "\u0000\u00b0\u00b1\u00037\u001b\u0000\u00b1\u00b2\u00037\u001b\u0000\u00b2"+
+        "6\u0001\u0000\u0000\u0000\u00b3\u00b4\u0007\n\u0000\u0000\u00b48\u0001"+
+        "\u0000\u0000\u0000\u000b\u0000N[afmu\u008a\u0090\u0096\u00a0\u0001\u0006"+
+        "\u0000\u0000";
+    public static final ATN _ATN =
+        new ATNDeserializer().deserialize(_serializedATN.toCharArray());
+    static {
+        _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
+        for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) {
+            _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i);
+        }
+    }
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseListener.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseListener.java
new file mode 100644
index 0000000000000..bce2044fa8175
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseListener.java
@@ -0,0 +1,194 @@
+// ANTLR GENERATED CODE: DO NOT EDIT
+package org.elasticsearch.xpack.kql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.antlr.v4.runtime.tree.ParseTreeListener;
+
+/**
+ * This interface defines a complete listener for a parse tree produced by
+ * {@link KqlBaseParser}.
+ */
+interface KqlBaseListener extends ParseTreeListener {
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#topLevelQuery}.
+     * @param ctx the parse tree
+     */
+    void enterTopLevelQuery(KqlBaseParser.TopLevelQueryContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#topLevelQuery}.
+     * @param ctx the parse tree
+     */
+    void exitTopLevelQuery(KqlBaseParser.TopLevelQueryContext ctx);
+    /**
+     * Enter a parse tree produced by the {@code notQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     */
+    void enterNotQuery(KqlBaseParser.NotQueryContext ctx);
+    /**
+     * Exit a parse tree produced by the {@code notQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     */
+    void exitNotQuery(KqlBaseParser.NotQueryContext ctx);
+    /**
+     * Enter a parse tree produced by the {@code booleanQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     */
+    void enterBooleanQuery(KqlBaseParser.BooleanQueryContext ctx);
+    /**
+     * Exit a parse tree produced by the {@code booleanQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     */
+    void exitBooleanQuery(KqlBaseParser.BooleanQueryContext ctx);
+    /**
+     * Enter a parse tree produced by the {@code defaultQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     */
+    void enterDefaultQuery(KqlBaseParser.DefaultQueryContext ctx);
+    /**
+     * Exit a parse tree produced by the {@code defaultQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     */
+    void exitDefaultQuery(KqlBaseParser.DefaultQueryContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#simpleQuery}.
+     * @param ctx the parse tree
+     */
+    void enterSimpleQuery(KqlBaseParser.SimpleQueryContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#simpleQuery}.
+     * @param ctx the parse tree
+     */
+    void exitSimpleQuery(KqlBaseParser.SimpleQueryContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#expression}.
+     * @param ctx the parse tree
+     */
+    void enterExpression(KqlBaseParser.ExpressionContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#expression}.
+     * @param ctx the parse tree
+     */
+    void exitExpression(KqlBaseParser.ExpressionContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#nestedQuery}.
+     * @param ctx the parse tree
+     */
+    void enterNestedQuery(KqlBaseParser.NestedQueryContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#nestedQuery}.
+     * @param ctx the parse tree
+     */
+    void exitNestedQuery(KqlBaseParser.NestedQueryContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#parenthesizedQuery}.
+     * @param ctx the parse tree
+     */
+    void enterParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#parenthesizedQuery}.
+     * @param ctx the parse tree
+     */
+    void exitParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#fieldRangeQuery}.
+     * @param ctx the parse tree
+     */
+    void enterFieldRangeQuery(KqlBaseParser.FieldRangeQueryContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#fieldRangeQuery}.
+     * @param ctx the parse tree
+     */
+    void exitFieldRangeQuery(KqlBaseParser.FieldRangeQueryContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#fieldTermQuery}.
+     * @param ctx the parse tree
+     */
+    void enterFieldTermQuery(KqlBaseParser.FieldTermQueryContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#fieldTermQuery}.
+     * @param ctx the parse tree
+     */
+    void exitFieldTermQuery(KqlBaseParser.FieldTermQueryContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#fieldName}.
+     * @param ctx the parse tree
+     */
+    void enterFieldName(KqlBaseParser.FieldNameContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#fieldName}.
+     * @param ctx the parse tree
+     */
+    void exitFieldName(KqlBaseParser.FieldNameContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#rangeQueryValue}.
+     * @param ctx the parse tree
+     */
+    void enterRangeQueryValue(KqlBaseParser.RangeQueryValueContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#rangeQueryValue}.
+     * @param ctx the parse tree
+     */
+    void exitRangeQueryValue(KqlBaseParser.RangeQueryValueContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#termQueryValue}.
+     * @param ctx the parse tree
+     */
+    void enterTermQueryValue(KqlBaseParser.TermQueryValueContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#termQueryValue}.
+     * @param ctx the parse tree
+     */
+    void exitTermQueryValue(KqlBaseParser.TermQueryValueContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#groupingTermExpression}.
+     * @param ctx the parse tree
+     */
+    void enterGroupingTermExpression(KqlBaseParser.GroupingTermExpressionContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#groupingTermExpression}.
+     * @param ctx the parse tree
+     */
+    void exitGroupingTermExpression(KqlBaseParser.GroupingTermExpressionContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#unquotedLiteralExpression}.
+     * @param ctx the parse tree
+     */
+    void enterUnquotedLiteralExpression(KqlBaseParser.UnquotedLiteralExpressionContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#unquotedLiteralExpression}.
+     * @param ctx the parse tree
+     */
+    void exitUnquotedLiteralExpression(KqlBaseParser.UnquotedLiteralExpressionContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#quotedStringExpression}.
+     * @param ctx the parse tree
+     */
+    void enterQuotedStringExpression(KqlBaseParser.QuotedStringExpressionContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#quotedStringExpression}.
+     * @param ctx the parse tree
+     */
+    void exitQuotedStringExpression(KqlBaseParser.QuotedStringExpressionContext ctx);
+    /**
+     * Enter a parse tree produced by {@link KqlBaseParser#wildcardExpression}.
+     * @param ctx the parse tree
+     */
+    void enterWildcardExpression(KqlBaseParser.WildcardExpressionContext ctx);
+    /**
+     * Exit a parse tree produced by {@link KqlBaseParser#wildcardExpression}.
+     * @param ctx the parse tree
+     */
+    void exitWildcardExpression(KqlBaseParser.WildcardExpressionContext ctx);
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java
new file mode 100644
index 0000000000000..3bd9cc4104d2c
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java
@@ -0,0 +1,1203 @@
+// ANTLR GENERATED CODE: DO NOT EDIT
+package org.elasticsearch.xpack.kql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.dfa.DFA;
+import org.antlr.v4.runtime.*;
+import org.antlr.v4.runtime.misc.*;
+import org.antlr.v4.runtime.tree.*;
+import java.util.List;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"})
+class KqlBaseParser extends Parser {
+    static { RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); }
+
+    protected static final DFA[] _decisionToDFA;
+    protected static final PredictionContextCache _sharedContextCache =
+        new PredictionContextCache();
+    public static final int
+        DEFAULT_SKIP=1, AND=2, OR=3, NOT=4, COLON=5, OP_COMPARE=6, LEFT_PARENTHESIS=7, 
+        RIGHT_PARENTHESIS=8, LEFT_CURLY_BRACKET=9, RIGHT_CURLY_BRACKET=10, UNQUOTED_LITERAL=11, 
+        QUOTED_STRING=12, WILDCARD=13;
+    public static final int
+        RULE_topLevelQuery = 0, RULE_query = 1, RULE_simpleQuery = 2, RULE_expression = 3, 
+        RULE_nestedQuery = 4, RULE_parenthesizedQuery = 5, RULE_fieldRangeQuery = 6, 
+        RULE_fieldTermQuery = 7, RULE_fieldName = 8, RULE_rangeQueryValue = 9, 
+        RULE_termQueryValue = 10, RULE_groupingTermExpression = 11, RULE_unquotedLiteralExpression = 12, 
+        RULE_quotedStringExpression = 13, RULE_wildcardExpression = 14;
+    private static String[] makeRuleNames() {
+        return new String[] {
+            "topLevelQuery", "query", "simpleQuery", "expression", "nestedQuery", 
+            "parenthesizedQuery", "fieldRangeQuery", "fieldTermQuery", "fieldName", 
+            "rangeQueryValue", "termQueryValue", "groupingTermExpression", "unquotedLiteralExpression", 
+            "quotedStringExpression", "wildcardExpression"
+        };
+    }
+    public static final String[] ruleNames = makeRuleNames();
+
+    private static String[] makeLiteralNames() {
+        return new String[] {
+            null, null, "'and'", "'or'", "'not'", "':'", null, "'('", "')'", "'{'", 
+            "'}'"
+        };
+    }
+    private static final String[] _LITERAL_NAMES = makeLiteralNames();
+    private static String[] makeSymbolicNames() {
+        return new String[] {
+            null, "DEFAULT_SKIP", "AND", "OR", "NOT", "COLON", "OP_COMPARE", "LEFT_PARENTHESIS", 
+            "RIGHT_PARENTHESIS", "LEFT_CURLY_BRACKET", "RIGHT_CURLY_BRACKET", "UNQUOTED_LITERAL", 
+            "QUOTED_STRING", "WILDCARD"
+        };
+    }
+    private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
+    public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
+
+    /**
+     * @deprecated Use {@link #VOCABULARY} instead.
+     */
+    @Deprecated
+    public static final String[] tokenNames;
+    static {
+        tokenNames = new String[_SYMBOLIC_NAMES.length];
+        for (int i = 0; i < tokenNames.length; i++) {
+            tokenNames[i] = VOCABULARY.getLiteralName(i);
+            if (tokenNames[i] == null) {
+                tokenNames[i] = VOCABULARY.getSymbolicName(i);
+            }
+
+            if (tokenNames[i] == null) {
+                tokenNames[i] = "<INVALID>";
+            }
+        }
+    }
+
+    @Override
+    @Deprecated
+    public String[] getTokenNames() {
+        return tokenNames;
+    }
+
+    @Override
+
+    public Vocabulary getVocabulary() {
+        return VOCABULARY;
+    }
+
+    @Override
+    public String getGrammarFileName() { return "KqlBase.g4"; }
+
+    @Override
+    public String[] getRuleNames() { return ruleNames; }
+
+    @Override
+    public String getSerializedATN() { return _serializedATN; }
+
+    @Override
+    public ATN getATN() { return _ATN; }
+
+    public KqlBaseParser(TokenStream input) {
+        super(input);
+        _interp = new ParserATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class TopLevelQueryContext extends ParserRuleContext {
+        public TerminalNode EOF() { return getToken(KqlBaseParser.EOF, 0); }
+        public QueryContext query() {
+            return getRuleContext(QueryContext.class,0);
+        }
+        public TopLevelQueryContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_topLevelQuery; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterTopLevelQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitTopLevelQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitTopLevelQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final TopLevelQueryContext topLevelQuery() throws RecognitionException {
+        TopLevelQueryContext _localctx = new TopLevelQueryContext(_ctx, getState());
+        enterRule(_localctx, 0, RULE_topLevelQuery);
+        int _la;
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(31);
+            _errHandler.sync(this);
+            _la = _input.LA(1);
+            if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 14480L) != 0)) {
+                {
+                setState(30);
+                query(0);
+                }
+            }
+
+            setState(33);
+            match(EOF);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class QueryContext extends ParserRuleContext {
+        public QueryContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_query; }
+     
+        public QueryContext() { }
+        public void copyFrom(QueryContext ctx) {
+            super.copyFrom(ctx);
+        }
+    }
+    @SuppressWarnings("CheckReturnValue")
+    public static class NotQueryContext extends QueryContext {
+        public SimpleQueryContext subQuery;
+        public TerminalNode NOT() { return getToken(KqlBaseParser.NOT, 0); }
+        public SimpleQueryContext simpleQuery() {
+            return getRuleContext(SimpleQueryContext.class,0);
+        }
+        public NotQueryContext(QueryContext ctx) { copyFrom(ctx); }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterNotQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitNotQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitNotQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+    @SuppressWarnings("CheckReturnValue")
+    public static class BooleanQueryContext extends QueryContext {
+        public List<QueryContext> query() {
+            return getRuleContexts(QueryContext.class);
+        }
+        public QueryContext query(int i) {
+            return getRuleContext(QueryContext.class,i);
+        }
+        public TerminalNode AND() { return getToken(KqlBaseParser.AND, 0); }
+        public TerminalNode OR() { return getToken(KqlBaseParser.OR, 0); }
+        public BooleanQueryContext(QueryContext ctx) { copyFrom(ctx); }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterBooleanQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitBooleanQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitBooleanQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+    @SuppressWarnings("CheckReturnValue")
+    public static class DefaultQueryContext extends QueryContext {
+        public SimpleQueryContext simpleQuery() {
+            return getRuleContext(SimpleQueryContext.class,0);
+        }
+        public DefaultQueryContext(QueryContext ctx) { copyFrom(ctx); }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterDefaultQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitDefaultQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitDefaultQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final QueryContext query() throws RecognitionException {
+        return query(0);
+    }
+
+    private QueryContext query(int _p) throws RecognitionException {
+        ParserRuleContext _parentctx = _ctx;
+        int _parentState = getState();
+        QueryContext _localctx = new QueryContext(_ctx, _parentState);
+        QueryContext _prevctx = _localctx;
+        int _startState = 2;
+        enterRecursionRule(_localctx, 2, RULE_query, _p);
+        int _la;
+        try {
+            int _alt;
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(39);
+            _errHandler.sync(this);
+            switch (_input.LA(1)) {
+            case NOT:
+                {
+                _localctx = new NotQueryContext(_localctx);
+                _ctx = _localctx;
+                _prevctx = _localctx;
+
+                setState(36);
+                match(NOT);
+                setState(37);
+                ((NotQueryContext)_localctx).subQuery = simpleQuery();
+                }
+                break;
+            case LEFT_PARENTHESIS:
+            case UNQUOTED_LITERAL:
+            case QUOTED_STRING:
+            case WILDCARD:
+                {
+                _localctx = new DefaultQueryContext(_localctx);
+                _ctx = _localctx;
+                _prevctx = _localctx;
+                setState(38);
+                simpleQuery();
+                }
+                break;
+            default:
+                throw new NoViableAltException(this);
+            }
+            _ctx.stop = _input.LT(-1);
+            setState(46);
+            _errHandler.sync(this);
+            _alt = getInterpreter().adaptivePredict(_input,2,_ctx);
+            while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+                if ( _alt==1 ) {
+                    if ( _parseListeners!=null ) triggerExitRuleEvent();
+                    _prevctx = _localctx;
+                    {
+                    {
+                    _localctx = new BooleanQueryContext(new QueryContext(_parentctx, _parentState));
+                    pushNewRecursionContext(_localctx, _startState, RULE_query);
+                    setState(41);
+                    if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
+                    setState(42);
+                    _la = _input.LA(1);
+                    if ( !(_la==AND || _la==OR) ) {
+                    _errHandler.recoverInline(this);
+                    }
+                    else {
+                        if ( _input.LA(1)==Token.EOF ) matchedEOF = true;
+                        _errHandler.reportMatch(this);
+                        consume();
+                    }
+                    setState(43);
+                    query(4);
+                    }
+                    } 
+                }
+                setState(48);
+                _errHandler.sync(this);
+                _alt = getInterpreter().adaptivePredict(_input,2,_ctx);
+            }
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            unrollRecursionContexts(_parentctx);
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class SimpleQueryContext extends ParserRuleContext {
+        public NestedQueryContext nestedQuery() {
+            return getRuleContext(NestedQueryContext.class,0);
+        }
+        public ExpressionContext expression() {
+            return getRuleContext(ExpressionContext.class,0);
+        }
+        public ParenthesizedQueryContext parenthesizedQuery() {
+            return getRuleContext(ParenthesizedQueryContext.class,0);
+        }
+        public SimpleQueryContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_simpleQuery; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterSimpleQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitSimpleQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitSimpleQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final SimpleQueryContext simpleQuery() throws RecognitionException {
+        SimpleQueryContext _localctx = new SimpleQueryContext(_ctx, getState());
+        enterRule(_localctx, 4, RULE_simpleQuery);
+        try {
+            setState(52);
+            _errHandler.sync(this);
+            switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) {
+            case 1:
+                enterOuterAlt(_localctx, 1);
+                {
+                setState(49);
+                nestedQuery();
+                }
+                break;
+            case 2:
+                enterOuterAlt(_localctx, 2);
+                {
+                setState(50);
+                expression();
+                }
+                break;
+            case 3:
+                enterOuterAlt(_localctx, 3);
+                {
+                setState(51);
+                parenthesizedQuery();
+                }
+                break;
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class ExpressionContext extends ParserRuleContext {
+        public FieldTermQueryContext fieldTermQuery() {
+            return getRuleContext(FieldTermQueryContext.class,0);
+        }
+        public FieldRangeQueryContext fieldRangeQuery() {
+            return getRuleContext(FieldRangeQueryContext.class,0);
+        }
+        public ExpressionContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_expression; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterExpression(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitExpression(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitExpression(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final ExpressionContext expression() throws RecognitionException {
+        ExpressionContext _localctx = new ExpressionContext(_ctx, getState());
+        enterRule(_localctx, 6, RULE_expression);
+        try {
+            setState(56);
+            _errHandler.sync(this);
+            switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
+            case 1:
+                enterOuterAlt(_localctx, 1);
+                {
+                setState(54);
+                fieldTermQuery();
+                }
+                break;
+            case 2:
+                enterOuterAlt(_localctx, 2);
+                {
+                setState(55);
+                fieldRangeQuery();
+                }
+                break;
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class NestedQueryContext extends ParserRuleContext {
+        public FieldNameContext fieldName() {
+            return getRuleContext(FieldNameContext.class,0);
+        }
+        public TerminalNode COLON() { return getToken(KqlBaseParser.COLON, 0); }
+        public TerminalNode LEFT_CURLY_BRACKET() { return getToken(KqlBaseParser.LEFT_CURLY_BRACKET, 0); }
+        public QueryContext query() {
+            return getRuleContext(QueryContext.class,0);
+        }
+        public TerminalNode RIGHT_CURLY_BRACKET() { return getToken(KqlBaseParser.RIGHT_CURLY_BRACKET, 0); }
+        public NestedQueryContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_nestedQuery; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterNestedQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitNestedQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitNestedQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final NestedQueryContext nestedQuery() throws RecognitionException {
+        NestedQueryContext _localctx = new NestedQueryContext(_ctx, getState());
+        enterRule(_localctx, 8, RULE_nestedQuery);
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(58);
+            fieldName();
+            setState(59);
+            match(COLON);
+            setState(60);
+            match(LEFT_CURLY_BRACKET);
+            setState(61);
+            query(0);
+            setState(62);
+            match(RIGHT_CURLY_BRACKET);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class ParenthesizedQueryContext extends ParserRuleContext {
+        public TerminalNode LEFT_PARENTHESIS() { return getToken(KqlBaseParser.LEFT_PARENTHESIS, 0); }
+        public QueryContext query() {
+            return getRuleContext(QueryContext.class,0);
+        }
+        public TerminalNode RIGHT_PARENTHESIS() { return getToken(KqlBaseParser.RIGHT_PARENTHESIS, 0); }
+        public ParenthesizedQueryContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_parenthesizedQuery; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterParenthesizedQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitParenthesizedQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitParenthesizedQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final ParenthesizedQueryContext parenthesizedQuery() throws RecognitionException {
+        ParenthesizedQueryContext _localctx = new ParenthesizedQueryContext(_ctx, getState());
+        enterRule(_localctx, 10, RULE_parenthesizedQuery);
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(64);
+            match(LEFT_PARENTHESIS);
+            setState(65);
+            query(0);
+            setState(66);
+            match(RIGHT_PARENTHESIS);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class FieldRangeQueryContext extends ParserRuleContext {
+        public Token operator;
+        public FieldNameContext fieldName() {
+            return getRuleContext(FieldNameContext.class,0);
+        }
+        public RangeQueryValueContext rangeQueryValue() {
+            return getRuleContext(RangeQueryValueContext.class,0);
+        }
+        public TerminalNode OP_COMPARE() { return getToken(KqlBaseParser.OP_COMPARE, 0); }
+        public FieldRangeQueryContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_fieldRangeQuery; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterFieldRangeQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitFieldRangeQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitFieldRangeQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final FieldRangeQueryContext fieldRangeQuery() throws RecognitionException {
+        FieldRangeQueryContext _localctx = new FieldRangeQueryContext(_ctx, getState());
+        enterRule(_localctx, 12, RULE_fieldRangeQuery);
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(68);
+            fieldName();
+            setState(69);
+            ((FieldRangeQueryContext)_localctx).operator = match(OP_COMPARE);
+            setState(70);
+            rangeQueryValue();
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class FieldTermQueryContext extends ParserRuleContext {
+        public TermQueryValueContext termQueryValue() {
+            return getRuleContext(TermQueryValueContext.class,0);
+        }
+        public FieldNameContext fieldName() {
+            return getRuleContext(FieldNameContext.class,0);
+        }
+        public TerminalNode COLON() { return getToken(KqlBaseParser.COLON, 0); }
+        public FieldTermQueryContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_fieldTermQuery; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterFieldTermQuery(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitFieldTermQuery(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitFieldTermQuery(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final FieldTermQueryContext fieldTermQuery() throws RecognitionException {
+        FieldTermQueryContext _localctx = new FieldTermQueryContext(_ctx, getState());
+        enterRule(_localctx, 14, RULE_fieldTermQuery);
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(75);
+            _errHandler.sync(this);
+            switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
+            case 1:
+                {
+                setState(72);
+                fieldName();
+                setState(73);
+                match(COLON);
+                }
+                break;
+            }
+            setState(77);
+            termQueryValue();
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class FieldNameContext extends ParserRuleContext {
+        public WildcardExpressionContext wildcardExpression() {
+            return getRuleContext(WildcardExpressionContext.class,0);
+        }
+        public UnquotedLiteralExpressionContext unquotedLiteralExpression() {
+            return getRuleContext(UnquotedLiteralExpressionContext.class,0);
+        }
+        public QuotedStringExpressionContext quotedStringExpression() {
+            return getRuleContext(QuotedStringExpressionContext.class,0);
+        }
+        public FieldNameContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_fieldName; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterFieldName(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitFieldName(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitFieldName(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final FieldNameContext fieldName() throws RecognitionException {
+        FieldNameContext _localctx = new FieldNameContext(_ctx, getState());
+        enterRule(_localctx, 16, RULE_fieldName);
+        try {
+            setState(82);
+            _errHandler.sync(this);
+            switch (_input.LA(1)) {
+            case WILDCARD:
+                enterOuterAlt(_localctx, 1);
+                {
+                setState(79);
+                wildcardExpression();
+                }
+                break;
+            case UNQUOTED_LITERAL:
+                enterOuterAlt(_localctx, 2);
+                {
+                setState(80);
+                unquotedLiteralExpression();
+                }
+                break;
+            case QUOTED_STRING:
+                enterOuterAlt(_localctx, 3);
+                {
+                setState(81);
+                quotedStringExpression();
+                }
+                break;
+            default:
+                throw new NoViableAltException(this);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class RangeQueryValueContext extends ParserRuleContext {
+        public UnquotedLiteralExpressionContext unquotedLiteralExpression() {
+            return getRuleContext(UnquotedLiteralExpressionContext.class,0);
+        }
+        public QuotedStringExpressionContext quotedStringExpression() {
+            return getRuleContext(QuotedStringExpressionContext.class,0);
+        }
+        public RangeQueryValueContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_rangeQueryValue; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterRangeQueryValue(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitRangeQueryValue(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitRangeQueryValue(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final RangeQueryValueContext rangeQueryValue() throws RecognitionException {
+        RangeQueryValueContext _localctx = new RangeQueryValueContext(_ctx, getState());
+        enterRule(_localctx, 18, RULE_rangeQueryValue);
+        try {
+            setState(86);
+            _errHandler.sync(this);
+            switch (_input.LA(1)) {
+            case UNQUOTED_LITERAL:
+                enterOuterAlt(_localctx, 1);
+                {
+                setState(84);
+                unquotedLiteralExpression();
+                }
+                break;
+            case QUOTED_STRING:
+                enterOuterAlt(_localctx, 2);
+                {
+                setState(85);
+                quotedStringExpression();
+                }
+                break;
+            default:
+                throw new NoViableAltException(this);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class TermQueryValueContext extends ParserRuleContext {
+        public UnquotedLiteralExpressionContext termValue;
+        public WildcardExpressionContext wildcardExpression() {
+            return getRuleContext(WildcardExpressionContext.class,0);
+        }
+        public QuotedStringExpressionContext quotedStringExpression() {
+            return getRuleContext(QuotedStringExpressionContext.class,0);
+        }
+        public UnquotedLiteralExpressionContext unquotedLiteralExpression() {
+            return getRuleContext(UnquotedLiteralExpressionContext.class,0);
+        }
+        public GroupingTermExpressionContext groupingTermExpression() {
+            return getRuleContext(GroupingTermExpressionContext.class,0);
+        }
+        public TermQueryValueContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_termQueryValue; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterTermQueryValue(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitTermQueryValue(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitTermQueryValue(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final TermQueryValueContext termQueryValue() throws RecognitionException {
+        TermQueryValueContext _localctx = new TermQueryValueContext(_ctx, getState());
+        enterRule(_localctx, 20, RULE_termQueryValue);
+        try {
+            setState(92);
+            _errHandler.sync(this);
+            switch (_input.LA(1)) {
+            case WILDCARD:
+                enterOuterAlt(_localctx, 1);
+                {
+                setState(88);
+                wildcardExpression();
+                }
+                break;
+            case QUOTED_STRING:
+                enterOuterAlt(_localctx, 2);
+                {
+                setState(89);
+                quotedStringExpression();
+                }
+                break;
+            case UNQUOTED_LITERAL:
+                enterOuterAlt(_localctx, 3);
+                {
+                setState(90);
+                ((TermQueryValueContext)_localctx).termValue = unquotedLiteralExpression();
+                }
+                break;
+            case LEFT_PARENTHESIS:
+                enterOuterAlt(_localctx, 4);
+                {
+                setState(91);
+                groupingTermExpression();
+                }
+                break;
+            default:
+                throw new NoViableAltException(this);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class GroupingTermExpressionContext extends ParserRuleContext {
+        public TerminalNode LEFT_PARENTHESIS() { return getToken(KqlBaseParser.LEFT_PARENTHESIS, 0); }
+        public UnquotedLiteralExpressionContext unquotedLiteralExpression() {
+            return getRuleContext(UnquotedLiteralExpressionContext.class,0);
+        }
+        public TerminalNode RIGHT_PARENTHESIS() { return getToken(KqlBaseParser.RIGHT_PARENTHESIS, 0); }
+        public GroupingTermExpressionContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_groupingTermExpression; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterGroupingTermExpression(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitGroupingTermExpression(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitGroupingTermExpression(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final GroupingTermExpressionContext groupingTermExpression() throws RecognitionException {
+        GroupingTermExpressionContext _localctx = new GroupingTermExpressionContext(_ctx, getState());
+        enterRule(_localctx, 22, RULE_groupingTermExpression);
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(94);
+            match(LEFT_PARENTHESIS);
+            setState(95);
+            unquotedLiteralExpression();
+            setState(96);
+            match(RIGHT_PARENTHESIS);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class UnquotedLiteralExpressionContext extends ParserRuleContext {
+        public List<TerminalNode> UNQUOTED_LITERAL() { return getTokens(KqlBaseParser.UNQUOTED_LITERAL); }
+        public TerminalNode UNQUOTED_LITERAL(int i) {
+            return getToken(KqlBaseParser.UNQUOTED_LITERAL, i);
+        }
+        public UnquotedLiteralExpressionContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_unquotedLiteralExpression; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterUnquotedLiteralExpression(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitUnquotedLiteralExpression(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitUnquotedLiteralExpression(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final UnquotedLiteralExpressionContext unquotedLiteralExpression() throws RecognitionException {
+        UnquotedLiteralExpressionContext _localctx = new UnquotedLiteralExpressionContext(_ctx, getState());
+        enterRule(_localctx, 24, RULE_unquotedLiteralExpression);
+        try {
+            int _alt;
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(99); 
+            _errHandler.sync(this);
+            _alt = 1;
+            do {
+                switch (_alt) {
+                case 1:
+                    {
+                    {
+                    setState(98);
+                    match(UNQUOTED_LITERAL);
+                    }
+                    }
+                    break;
+                default:
+                    throw new NoViableAltException(this);
+                }
+                setState(101); 
+                _errHandler.sync(this);
+                _alt = getInterpreter().adaptivePredict(_input,9,_ctx);
+            } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class QuotedStringExpressionContext extends ParserRuleContext {
+        public TerminalNode QUOTED_STRING() { return getToken(KqlBaseParser.QUOTED_STRING, 0); }
+        public QuotedStringExpressionContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_quotedStringExpression; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterQuotedStringExpression(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitQuotedStringExpression(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitQuotedStringExpression(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final QuotedStringExpressionContext quotedStringExpression() throws RecognitionException {
+        QuotedStringExpressionContext _localctx = new QuotedStringExpressionContext(_ctx, getState());
+        enterRule(_localctx, 26, RULE_quotedStringExpression);
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(103);
+            match(QUOTED_STRING);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    @SuppressWarnings("CheckReturnValue")
+    public static class WildcardExpressionContext extends ParserRuleContext {
+        public TerminalNode WILDCARD() { return getToken(KqlBaseParser.WILDCARD, 0); }
+        public WildcardExpressionContext(ParserRuleContext parent, int invokingState) {
+            super(parent, invokingState);
+        }
+        @Override public int getRuleIndex() { return RULE_wildcardExpression; }
+        @Override
+        public void enterRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).enterWildcardExpression(this);
+        }
+        @Override
+        public void exitRule(ParseTreeListener listener) {
+            if ( listener instanceof KqlBaseListener ) ((KqlBaseListener)listener).exitWildcardExpression(this);
+        }
+        @Override
+        public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+            if ( visitor instanceof KqlBaseVisitor ) return ((KqlBaseVisitor<? extends T>)visitor).visitWildcardExpression(this);
+            else return visitor.visitChildren(this);
+        }
+    }
+
+    public final WildcardExpressionContext wildcardExpression() throws RecognitionException {
+        WildcardExpressionContext _localctx = new WildcardExpressionContext(_ctx, getState());
+        enterRule(_localctx, 28, RULE_wildcardExpression);
+        try {
+            enterOuterAlt(_localctx, 1);
+            {
+            setState(105);
+            match(WILDCARD);
+            }
+        }
+        catch (RecognitionException re) {
+            _localctx.exception = re;
+            _errHandler.reportError(this, re);
+            _errHandler.recover(this, re);
+        }
+        finally {
+            exitRule();
+        }
+        return _localctx;
+    }
+
+    public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
+        switch (ruleIndex) {
+        case 1:
+            return query_sempred((QueryContext)_localctx, predIndex);
+        }
+        return true;
+    }
+    private boolean query_sempred(QueryContext _localctx, int predIndex) {
+        switch (predIndex) {
+        case 0:
+            return precpred(_ctx, 3);
+        }
+        return true;
+    }
+
+    public static final String _serializedATN =
+        "\u0004\u0001\rl\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
+        "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+
+        "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+
+        "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+
+        "\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0001\u0000\u0003\u0000"+
+        " \b\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001"+
+        "\u0001\u0001\u0003\u0001(\b\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
+        "\u0005\u0001-\b\u0001\n\u0001\f\u00010\t\u0001\u0001\u0002\u0001\u0002"+
+        "\u0001\u0002\u0003\u00025\b\u0002\u0001\u0003\u0001\u0003\u0003\u0003"+
+        "9\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+
+        "\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006"+
+        "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007\u0001\u0007"+
+        "\u0003\u0007L\b\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b"+
+        "\u0003\bS\b\b\u0001\t\u0001\t\u0003\tW\b\t\u0001\n\u0001\n\u0001\n\u0001"+
+        "\n\u0003\n]\b\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+
+        "\f\u0004\fd\b\f\u000b\f\f\fe\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001"+
+        "\u000e\u0000\u0001\u0002\u000f\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+
+        "\u0012\u0014\u0016\u0018\u001a\u001c\u0000\u0001\u0001\u0000\u0002\u0003"+
+        "j\u0000\u001f\u0001\u0000\u0000\u0000\u0002\'\u0001\u0000\u0000\u0000"+
+        "\u00044\u0001\u0000\u0000\u0000\u00068\u0001\u0000\u0000\u0000\b:\u0001"+
+        "\u0000\u0000\u0000\n@\u0001\u0000\u0000\u0000\fD\u0001\u0000\u0000\u0000"+
+        "\u000eK\u0001\u0000\u0000\u0000\u0010R\u0001\u0000\u0000\u0000\u0012V"+
+        "\u0001\u0000\u0000\u0000\u0014\\\u0001\u0000\u0000\u0000\u0016^\u0001"+
+        "\u0000\u0000\u0000\u0018c\u0001\u0000\u0000\u0000\u001ag\u0001\u0000\u0000"+
+        "\u0000\u001ci\u0001\u0000\u0000\u0000\u001e \u0003\u0002\u0001\u0000\u001f"+
+        "\u001e\u0001\u0000\u0000\u0000\u001f \u0001\u0000\u0000\u0000 !\u0001"+
+        "\u0000\u0000\u0000!\"\u0005\u0000\u0000\u0001\"\u0001\u0001\u0000\u0000"+
+        "\u0000#$\u0006\u0001\uffff\uffff\u0000$%\u0005\u0004\u0000\u0000%(\u0003"+
+        "\u0004\u0002\u0000&(\u0003\u0004\u0002\u0000\'#\u0001\u0000\u0000\u0000"+
+        "\'&\u0001\u0000\u0000\u0000(.\u0001\u0000\u0000\u0000)*\n\u0003\u0000"+
+        "\u0000*+\u0007\u0000\u0000\u0000+-\u0003\u0002\u0001\u0004,)\u0001\u0000"+
+        "\u0000\u0000-0\u0001\u0000\u0000\u0000.,\u0001\u0000\u0000\u0000./\u0001"+
+        "\u0000\u0000\u0000/\u0003\u0001\u0000\u0000\u00000.\u0001\u0000\u0000"+
+        "\u000015\u0003\b\u0004\u000025\u0003\u0006\u0003\u000035\u0003\n\u0005"+
+        "\u000041\u0001\u0000\u0000\u000042\u0001\u0000\u0000\u000043\u0001\u0000"+
+        "\u0000\u00005\u0005\u0001\u0000\u0000\u000069\u0003\u000e\u0007\u0000"+
+        "79\u0003\f\u0006\u000086\u0001\u0000\u0000\u000087\u0001\u0000\u0000\u0000"+
+        "9\u0007\u0001\u0000\u0000\u0000:;\u0003\u0010\b\u0000;<\u0005\u0005\u0000"+
+        "\u0000<=\u0005\t\u0000\u0000=>\u0003\u0002\u0001\u0000>?\u0005\n\u0000"+
+        "\u0000?\t\u0001\u0000\u0000\u0000@A\u0005\u0007\u0000\u0000AB\u0003\u0002"+
+        "\u0001\u0000BC\u0005\b\u0000\u0000C\u000b\u0001\u0000\u0000\u0000DE\u0003"+
+        "\u0010\b\u0000EF\u0005\u0006\u0000\u0000FG\u0003\u0012\t\u0000G\r\u0001"+
+        "\u0000\u0000\u0000HI\u0003\u0010\b\u0000IJ\u0005\u0005\u0000\u0000JL\u0001"+
+        "\u0000\u0000\u0000KH\u0001\u0000\u0000\u0000KL\u0001\u0000\u0000\u0000"+
+        "LM\u0001\u0000\u0000\u0000MN\u0003\u0014\n\u0000N\u000f\u0001\u0000\u0000"+
+        "\u0000OS\u0003\u001c\u000e\u0000PS\u0003\u0018\f\u0000QS\u0003\u001a\r"+
+        "\u0000RO\u0001\u0000\u0000\u0000RP\u0001\u0000\u0000\u0000RQ\u0001\u0000"+
+        "\u0000\u0000S\u0011\u0001\u0000\u0000\u0000TW\u0003\u0018\f\u0000UW\u0003"+
+        "\u001a\r\u0000VT\u0001\u0000\u0000\u0000VU\u0001\u0000\u0000\u0000W\u0013"+
+        "\u0001\u0000\u0000\u0000X]\u0003\u001c\u000e\u0000Y]\u0003\u001a\r\u0000"+
+        "Z]\u0003\u0018\f\u0000[]\u0003\u0016\u000b\u0000\\X\u0001\u0000\u0000"+
+        "\u0000\\Y\u0001\u0000\u0000\u0000\\Z\u0001\u0000\u0000\u0000\\[\u0001"+
+        "\u0000\u0000\u0000]\u0015\u0001\u0000\u0000\u0000^_\u0005\u0007\u0000"+
+        "\u0000_`\u0003\u0018\f\u0000`a\u0005\b\u0000\u0000a\u0017\u0001\u0000"+
+        "\u0000\u0000bd\u0005\u000b\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+
+        "\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+
+        "f\u0019\u0001\u0000\u0000\u0000gh\u0005\f\u0000\u0000h\u001b\u0001\u0000"+
+        "\u0000\u0000ij\u0005\r\u0000\u0000j\u001d\u0001\u0000\u0000\u0000\n\u001f"+
+        "\'.48KRV\\e";
+    public static final ATN _ATN =
+        new ATNDeserializer().deserialize(_serializedATN.toCharArray());
+    static {
+        _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
+        for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) {
+            _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i);
+        }
+    }
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseVisitor.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseVisitor.java
new file mode 100644
index 0000000000000..55fa21f0e899d
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseVisitor.java
@@ -0,0 +1,126 @@
+// ANTLR GENERATED CODE: DO NOT EDIT
+package org.elasticsearch.xpack.kql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.antlr.v4.runtime.tree.ParseTreeVisitor;
+
+/**
+ * This interface defines a complete generic visitor for a parse tree produced
+ * by {@link KqlBaseParser}.
+ *
+ * @param <T> The return type of the visit operation. Use {@link Void} for
+ * operations with no return type.
+ */
+interface KqlBaseVisitor<T> extends ParseTreeVisitor<T> {
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#topLevelQuery}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitTopLevelQuery(KqlBaseParser.TopLevelQueryContext ctx);
+    /**
+     * Visit a parse tree produced by the {@code notQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitNotQuery(KqlBaseParser.NotQueryContext ctx);
+    /**
+     * Visit a parse tree produced by the {@code booleanQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitBooleanQuery(KqlBaseParser.BooleanQueryContext ctx);
+    /**
+     * Visit a parse tree produced by the {@code defaultQuery}
+     * labeled alternative in {@link KqlBaseParser#query}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitDefaultQuery(KqlBaseParser.DefaultQueryContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#simpleQuery}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitSimpleQuery(KqlBaseParser.SimpleQueryContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#expression}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitExpression(KqlBaseParser.ExpressionContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#nestedQuery}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitNestedQuery(KqlBaseParser.NestedQueryContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#parenthesizedQuery}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#fieldRangeQuery}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitFieldRangeQuery(KqlBaseParser.FieldRangeQueryContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#fieldTermQuery}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitFieldTermQuery(KqlBaseParser.FieldTermQueryContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#fieldName}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitFieldName(KqlBaseParser.FieldNameContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#rangeQueryValue}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitRangeQueryValue(KqlBaseParser.RangeQueryValueContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#termQueryValue}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitTermQueryValue(KqlBaseParser.TermQueryValueContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#groupingTermExpression}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitGroupingTermExpression(KqlBaseParser.GroupingTermExpressionContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#unquotedLiteralExpression}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitUnquotedLiteralExpression(KqlBaseParser.UnquotedLiteralExpressionContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#quotedStringExpression}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitQuotedStringExpression(KqlBaseParser.QuotedStringExpressionContext ctx);
+    /**
+     * Visit a parse tree produced by {@link KqlBaseParser#wildcardExpression}.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    T visitWildcardExpression(KqlBaseParser.WildcardExpressionContext ctx);
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParser.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParser.java
new file mode 100644
index 0000000000000..41bda7524c653
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParser.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.kql.parser;
+
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.atn.PredictionMode;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+public class KqlParser {
+    private static final Logger log = LogManager.getLogger(KqlParser.class);
+
+    public QueryBuilder parseKqlQuery(String kqlQuery, SearchExecutionContext searchExecutionContext) {
+        if (log.isDebugEnabled()) {
+            log.debug("Parsing KQL query: {}", kqlQuery);
+        }
+
+        return invokeParser(kqlQuery, searchExecutionContext, KqlBaseParser::topLevelQuery, KqlAstBuilder::toQueryBuilder);
+    }
+
+    private <T> T invokeParser(
+        String kqlQuery,
+        SearchExecutionContext searchExecutionContext,
+        Function<KqlBaseParser, ParserRuleContext> parseFunction,
+        BiFunction<KqlAstBuilder, ParserRuleContext, T> visitor
+    ) {
+        KqlBaseLexer lexer = new KqlBaseLexer(CharStreams.fromString(kqlQuery));
+
+        lexer.removeErrorListeners();
+        lexer.addErrorListener(ERROR_LISTENER);
+
+        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
+        KqlBaseParser parser = new KqlBaseParser(tokenStream);
+
+        parser.removeErrorListeners();
+        parser.addErrorListener(ERROR_LISTENER);
+
+        parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
+
+        ParserRuleContext tree = parseFunction.apply(parser);
+
+        if (log.isTraceEnabled()) {
+            log.trace("Parse tree: {}", tree.toStringTree());
+        }
+
+        return visitor.apply(new KqlAstBuilder(searchExecutionContext), tree);
+    }
+
+    private static final BaseErrorListener ERROR_LISTENER = new BaseErrorListener() {
+        @Override
+        public void syntaxError(
+            Recognizer<?, ?> recognizer,
+            Object offendingSymbol,
+            int line,
+            int charPositionInLine,
+            String message,
+            RecognitionException e
+        ) {
+            throw new KqlParsingException(message, line, charPositionInLine, e);
+        }
+    };
+}
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingException.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingException.java
new file mode 100644
index 0000000000000..e6ac9020174f2
--- /dev/null
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingException.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.kql.parser;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.rest.RestStatus;
+
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+
+public class KqlParsingException extends ElasticsearchException {
+
+    private final int line;
+    private final int charPositionInLine;
+
+    public KqlParsingException(String message, Exception cause, int line, int charPositionInLine) {
+        super(message, cause);
+        this.line = line;
+        this.charPositionInLine = charPositionInLine;
+    }
+
+    public KqlParsingException(String message, int line, int charPositionInLine, Object... args) {
+        super(message, args);
+        this.line = line;
+        this.charPositionInLine = charPositionInLine;
+    }
+
+    public KqlParsingException(String message, Throwable cause, int line, int charPositionInLine, Object... args) {
+        super(message, cause, args);
+        this.line = line;
+        this.charPositionInLine = charPositionInLine;
+    }
+
+    public int getLineNumber() {
+        return line;
+    }
+
+    public int getColumnNumber() {
+        return charPositionInLine + 1;
+    }
+
+    public String getErrorMessage() {
+        return super.getMessage();
+    }
+
+    @Override
+    public String getMessage() {
+        return format("line {}:{}: {}", getLineNumber(), getColumnNumber(), getErrorMessage());
+    }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.BAD_REQUEST;
+    }
+}
diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserTests.java
new file mode 100644
index 0000000000000..58b162409412d
--- /dev/null
+++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserTests.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.kql.parser;
+
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.test.AbstractBuilderTestCase;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.isA;
+
+public class KqlParserTests extends AbstractBuilderTestCase {
+
+    public void testEmptyQueryParsing() {
+        KqlParser parser = new KqlParser();
+        SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
+        assertThat(parser.parseKqlQuery("", searchExecutionContext), isA(MatchAllQueryBuilder.class));
+    }
+
+    public void testSupportedQueries() throws Exception {
+        KqlParser parser = new KqlParser();
+        SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
+
+        for (String query : readQueries("/supported-queries")) {
+            try {
+                parser.parseKqlQuery(query, searchExecutionContext);
+            } catch (Throwable e) {
+                throw new AssertionError("Unexpected error during query parsing [ " + query + "]", e);
+            }
+        }
+    }
+
+    public void testUnsupportedQueries() throws Exception {
+        KqlParser parser = new KqlParser();
+        SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
+
+        for (String query : readQueries("/unsupported-queries")) {
+            assertThrows(
+                "Was expecting a KqlParsingException exception to be thrown while parsing query [" + query + "]",
+                KqlParsingException.class,
+                () -> parser.parseKqlQuery(query, searchExecutionContext)
+            );
+        }
+    }
+
+    public void testSyntaxErrorsHandling() {
+        KqlParser parser = new KqlParser();
+        SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
+
+        {
+            KqlParsingException e = assertThrows(
+                KqlParsingException.class,
+                () -> parser.parseKqlQuery("foo: \"bar", searchExecutionContext)
+            );
+            assertThat(e.getLineNumber(), equalTo(1));
+            assertThat(e.getColumnNumber(), equalTo(6));
+            assertThat(e.getMessage(), equalTo("line 1:6: token recognition error at: '\"bar'"));
+        }
+
+        {
+            KqlParsingException e = assertThrows(
+                KqlParsingException.class,
+                () -> parser.parseKqlQuery("foo: (bar baz AND qux", searchExecutionContext)
+            );
+            assertThat(e.getLineNumber(), equalTo(1));
+            assertThat(e.getColumnNumber(), equalTo(15));
+            assertThat(e.getMessage(), equalTo("line 1:15: missing ')' at 'AND'"));
+        }
+    }
+
+    private static List<String> readQueries(String source) throws Exception {
+        URL url = KqlParserTests.class.getResource(source);
+        Objects.requireNonNull(source, "Cannot find resource " + url);
+
+        List<String> queries = new ArrayList<>();
+
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(readFromJarUrl(url), StandardCharsets.UTF_8))) {
+            String line;
+
+            while ((line = reader.readLine()) != null) {
+                String query = line.trim();
+                // ignore comments
+                if (query.isEmpty() == false && query.startsWith("//") == false) {
+                    queries.add(line.trim());
+                }
+            }
+        }
+        return queries;
+    }
+
+    @SuppressForbidden(reason = "test reads from jar")
+    private static InputStream readFromJarUrl(URL source) throws IOException {
+        URLConnection con = source.openConnection();
+        // do not to cache files (to avoid keeping file handles around)
+        con.setUseCaches(false);
+        return con.getInputStream();
+    }
+}
diff --git a/x-pack/plugin/kql/src/test/resources/supported-queries b/x-pack/plugin/kql/src/test/resources/supported-queries
new file mode 100644
index 0000000000000..d750f16149112
--- /dev/null
+++ b/x-pack/plugin/kql/src/test/resources/supported-queries
@@ -0,0 +1,111 @@
+// Match all queries
+*
+*:*
+(*:*)
+
+// Queries with no field
+200
+foo
+foo bar
+(foo bar)
+foo*
+*foo
+f*oo
+"foo bar"
+
+// Queries with all fields
+ *:200
+ *:foo
+ *:foo bar
+ *:foo*
+ *:f*oo
+ *: *foo
+ *:"foo bar"
+
+// Querying a field
+foo_field:200
+foo_field:foo
+foo_field:foo bar
+foo_field:(foo bar)
+foo_field:foo*
+foo_field: f*oo
+foo_field: *foo
+foo_field:"foo bar"
+foo_field.subfield:foo
+foo_*_field:foo
+foo_field:*
+foo_*:*
+
+// Range queries
+foo_field<200
+foo_field<foo
+foo_field<"foo bar"
+foo_field>=200
+foo_field>=foo
+foo_field>"foo bar"
+foo_field<=foo
+foo_field>=foo
+
+// Boolean queries
+NOT foo
+NOT foo bar
+NOT foo_field:foo
+NOT foo_field<foo
+foo_field:foo AND foo_field:foo bar
+foo_field<foo AND foo_field>bar
+(foo_field:foo) AND (foo_field:foo bar)
+foo_field:foo OR foo_field:foo bar
+NOT(foo_field:foo OR foo_field:foo bar)
+NOT(foo_field:foo AND foo_field:foo bar)
+NOT foo_field:foo AND NOT foo_field:foo bar
+(NOT foo_field:foo) AND (NOT foo_field:foo bar)
+NOT(foo_field:foo) AND NOT(foo_field:foo bar)
+foo_field:foo AND foo_field:foo bar AND foo bar
+foo_field:foo AND foo_field:foo bar OR foo bar
+foo_field:foo OR foo_field:foo bar OR foo bar
+foo_field:foo OR foo_field:foo bar AND foo bar
+foo_field:foo AND (foo_field:foo bar OR foo bar)
+foo_field:foo AND (foo_field:foo bar OR foo bar)
+foo_field:foo OR (foo_field:foo bar OR foo bar)
+
+// Nested queries
+nested_field: { NOT foo }
+nested_field: { NOT foo bar }
+nested_field: { NOT foo_field:foo }
+nested_field: { foo_field:foo AND foo_field:foo bar }
+nested_field: { foo_field<foo AND foo_field>bar }
+nested_field: { (foo_field:foo) AND (foo_field:foo bar) }
+nested_field: { foo_field:foo OR foo_field:foo bar }
+nested_field: { NOT(foo_field:foo OR foo_field:foo bar) }
+nested_field: { NOT(foo_field:foo AND foo_field:foo bar) }
+nested_field: { NOT foo_field:foo AND NOT foo_field:foo bar }
+nested_field: { (NOT foo_field:foo) AND (NOT foo_field:foo bar) }
+nested_field: { NOT(foo_field:foo) AND NOT(foo_field:foo bar) }
+nested_field: { foo_field:foo AND foo_field:foo bar AND foo bar }
+nested_field: { foo_field:foo AND foo_field:foo bar OR foo bar }
+nested_field: { foo_field:foo OR foo_field:foo bar OR foo bar }
+nested_field: { foo_field:foo OR foo_field:foo bar AND foo bar }
+nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) }
+nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) }
+nested_field: { foo_field:foo OR (foo_field:foo bar OR foo bar) }
+nested_field: { sub_nested_field : { foo_field:foo } AND foo_field:foo bar }
+
+// Queries with escape sequences
+foo_field : (foo\(bar\))
+foo_field : foo\:bar
+foo_field : (foo \and bar)
+foo_field : (foo \or bar)
+foo_field : foo \not bar
+foo_field : foo \{bar\}
+foo_field : foo \(bar\)
+foo_field : foo \\ bar
+foo_field : foo \"bar\"
+
+foo_field : "foo and bar"
+foo_field : "foo not bar"
+foo_field : "foo or bar"
+foo_field : "foo : bar"
+foo_field : "foo { bar }"
+foo_field : "foo (bar)"
+foo_field : "foo \\ bar"
+foo_field : "foo \"bar\""
diff --git a/x-pack/plugin/kql/src/test/resources/unsupported-queries b/x-pack/plugin/kql/src/test/resources/unsupported-queries
new file mode 100644
index 0000000000000..545b03576b331
--- /dev/null
+++ b/x-pack/plugin/kql/src/test/resources/unsupported-queries
@@ -0,0 +1,41 @@
+
+// Incomplete expressions
+foo_field :
+foo_field <
+foo_field >
+foo_field >=
+foo_field <=
+
+// Parentheses mismatch
+foo_field: (foo bar
+foo_field: foo bar)
+NOT foo_field:foo OR foo_field:foo bar)
+NOT (foo_field:foo AND) foo_field:foo bar
+
+// Quotes mismatch
+foo_field: "foo bar
+foo_field: foo bar"
+
+
+// Invalid boolean queries
+foo AND
+AND foo
+foo OR
+OR foo
+NOT foo:
+
+// Can't nest grouping terms parentheses
+foo_field:(foo (bar))
+
+// Bad syntax for nested fields:
+nested_field { foo: bar }
+
+// Missing escape sequences:
+foo_field: foo:bar
+foo_field: (foo and bar)
+foo_field: (foo or bar)
+foo_field: foo not bar
+foo_field: foo { bar }
+foo_field: foo (bar)
+foo_field: foo "bar"
+foo_field: "foo "bar""
diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle
index 929d7dad2f5e6..60578f832d153 100644
--- a/x-pack/plugin/logsdb/build.gradle
+++ b/x-pack/plugin/logsdb/build.gradle
@@ -25,7 +25,7 @@ base {
 
 restResources {
   restApi {
-    include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities'
+    include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities', 'esql.query'
   }
 }
 
diff --git a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
deleted file mode 100644
index 813a181045f2e..0000000000000
--- a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.logsdb;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.hamcrest.Matchers;
-import org.junit.ClassRule;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-public class LogsdbRestIT extends ESRestTestCase {
-
-    @ClassRule
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
-        .setting("xpack.license.self_generated.type", "basic")
-        .setting("xpack.security.enabled", "false")
-        .build();
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    public void testFeatureUsageWithLogsdbIndex() throws IOException {
-        {
-            var response = getAsMap("/_license/feature_usage");
-            @SuppressWarnings("unchecked")
-            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
-            assertThat(features, Matchers.empty());
-        }
-        {
-            if (randomBoolean()) {
-                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
-            } else if (randomBoolean()) {
-                String mapping = """
-                    {
-                        "properties": {
-                            "field1": {
-                                "type": "keyword",
-                                "time_series_dimension": true
-                            }
-                        }
-                    }
-                    """;
-                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
-                createIndex("test-index", settings, mapping);
-            } else {
-                String mapping = """
-                    {
-                        "_source": {
-                            "mode": "synthetic"
-                        }
-                    }
-                    """;
-                createIndex("test-index", Settings.EMPTY, mapping);
-            }
-            var response = getAsMap("/_license/feature_usage");
-            @SuppressWarnings("unchecked")
-            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
-            assertThat(features, Matchers.empty());
-        }
-    }
-
-}
diff --git a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java
new file mode 100644
index 0000000000000..381c83ceee289
--- /dev/null
+++ b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class LogsdbWithBasicRestIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.license.self_generated.type", "basic")
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public void testFeatureUsageWithLogsdbIndex() throws IOException {
+        {
+            var response = getAsMap("/_license/feature_usage");
+            @SuppressWarnings("unchecked")
+            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
+            assertThat(features, Matchers.empty());
+        }
+        {
+            if (randomBoolean()) {
+                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            } else if (randomBoolean()) {
+                String mapping = """
+                    {
+                        "properties": {
+                            "field1": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            }
+                        }
+                    }
+                    """;
+                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
+                createIndex("test-index", settings, mapping);
+            } else {
+                String mapping = """
+                    {
+                        "_source": {
+                            "mode": "synthetic"
+                        }
+                    }
+                    """;
+                createIndex("test-index", Settings.EMPTY, mapping);
+            }
+            var response = getAsMap("/_license/feature_usage");
+            @SuppressWarnings("unchecked")
+            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
+            assertThat(features, Matchers.empty());
+        }
+    }
+
+    public void testLogsdbIndexGetsStoredSource() throws IOException {
+        final String index = "test-index";
+        createIndex(index, Settings.builder().put("index.mode", "logsdb").build());
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+    }
+
+    public void testLogsdbOverrideSyntheticSourceModeInMapping() throws IOException {
+        final String index = "test-index";
+        String mapping = """
+            {
+                "_source": {
+                    "mode": "synthetic"
+                }
+            }
+            """;
+        createIndex(index, Settings.builder().put("index.mode", "logsdb").build(), mapping);
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+    }
+
+    public void testLogsdbOverrideSyntheticSourceSetting() throws IOException {
+        final String index = "test-index";
+        createIndex(
+            index,
+            Settings.builder().put("index.mode", "logsdb").put("index.mapping.source.mode", SourceFieldMapper.Mode.SYNTHETIC).build()
+        );
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+    }
+
+    public void testLogsdbOverrideNullSyntheticSourceSetting() throws IOException {
+        final String index = "test-index";
+        createIndex(index, Settings.builder().put("index.mode", "logsdb").putNull("index.mapping.source.mode").build());
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+    }
+
+    public void testLogsdbOverrideSyntheticSourceSettingInTemplate() throws IOException {
+        var request = new Request("POST", "/_index_template/1");
+        request.setJsonEntity("""
+            {
+                "index_patterns": ["test-*"],
+                "template": {
+                    "settings":{
+                        "index": {
+                            "mode": "logsdb",
+                            "mapping": {
+                              "source": {
+                                "mode": "synthetic"
+                              }
+                            }
+                        }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        final String index = "test-index";
+        createIndex(index);
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+    }
+
+    public void testLogsdbOverrideNullInTemplate() throws IOException {
+        var request = new Request("POST", "/_index_template/1");
+        request.setJsonEntity("""
+            {
+                "index_patterns": ["test-*"],
+                "template": {
+                    "settings":{
+                        "index": {
+                            "mode": "logsdb",
+                            "mapping": {
+                              "source": {
+                                "mode": null
+                              }
+                            }
+                        }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        final String index = "test-index";
+        createIndex(index);
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+    }
+
+    public void testLogsdbOverrideDefaultModeForLogsIndex() throws IOException {
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }");
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/_index_template/1");
+        request.setJsonEntity("""
+            {
+                "index_patterns": ["logs-test-*"],
+                "data_stream": {
+                }
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/logs-test-foo/_doc");
+        request.setJsonEntity("""
+            {
+                "@timestamp": "2020-01-01T00:00:00.000Z",
+                "host.name": "foo",
+                "message": "bar"
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        String index = DataStream.getDefaultBackingIndexName("logs-test-foo", 1);
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+    }
+}
diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsDbSourceModeMigrationIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsDbSourceModeMigrationIT.java
new file mode 100644
index 0000000000000..adb23567e3933
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsDbSourceModeMigrationIT.java
@@ -0,0 +1,290 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+
+public class LogsDbSourceModeMigrationIT extends LogsIndexModeRestTestIT {
+    public static final String INDEX_TEMPLATE = """
+        {
+          "index_patterns": ["my-logs-*-*"],
+          "priority": 100,
+          "data_stream": {},
+          "composed_of": [
+            "my-logs-mapping",
+            "my-logs-original-source",
+            "my-logs-migrated-source"
+          ],
+          "ignore_missing_component_templates": ["my-logs-original-source", "my-logs-migrated-source"]
+        }
+        """;
+
+    public static final String MAPPING_COMPONENT_TEMPLATE = """
+        {
+          "template": {
+            "settings": {
+              "index": {
+                "mode": "logsdb"
+              }
+            },
+            "mappings": {
+              "properties": {
+                "@timestamp": {
+                  "type": "date",
+                  "format": "epoch_millis"
+                },
+                "message": {
+                  "type": "text"
+                },
+                "method": {
+                  "type": "keyword"
+                },
+                "hits": {
+                  "type": "long"
+                }
+              }
+            }
+          }
+        }""";
+
+    public static final String STORED_SOURCE_COMPONENT_TEMPLATE = """
+        {
+          "template": {
+            "settings": {
+              "index": {
+                "mapping.source.mode": "stored"
+              }
+            }
+          }
+        }""";
+
+    public static final String SYNTHETIC_SOURCE_COMPONENT_TEMPLATE = """
+        {
+          "template": {
+            "settings": {
+              "index": {
+                "mapping.source.mode": "synthetic"
+              }
+            }
+          }
+        }""";
+
+    @ClassRule()
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .module("constant-keyword")
+        .module("data-streams")
+        .module("mapper-extras")
+        .module("x-pack-aggregate-metric")
+        .module("x-pack-stack")
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.otel_data.registry.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("cluster.logsdb.enabled", "true")
+        .setting("stack.templates.enabled", "false")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Before
+    public void setup() {
+        client = client();
+    }
+
+    private RestClient client;
+
+    public void testSwitchFromStoredToSyntheticSource() throws IOException {
+        assertOK(putComponentTemplate(client, "my-logs-mapping", MAPPING_COMPONENT_TEMPLATE));
+        assertOK(putComponentTemplate(client, "my-logs-original-source", STORED_SOURCE_COMPONENT_TEMPLATE));
+
+        assertOK(putTemplate(client, "my-logs", INDEX_TEMPLATE));
+        assertOK(createDataStream(client, "my-logs-ds-test"));
+
+        var initialSourceMode = (String) getSetting(
+            client,
+            getDataStreamBackingIndex(client, "my-logs-ds-test", 0),
+            "index.mapping.source.mode"
+        );
+        assertThat(initialSourceMode, equalTo("stored"));
+        var initialIndexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "my-logs-ds-test", 0), "index.mode");
+        assertThat(initialIndexMode, equalTo("logsdb"));
+
+        var indexedWithStoredSource = new ArrayList<XContentBuilder>();
+        var indexedWithSyntheticSource = new ArrayList<XContentBuilder>();
+        for (int i = 0; i < 10; i++) {
+            indexedWithStoredSource.add(generateDoc());
+            indexedWithSyntheticSource.add(generateDoc());
+        }
+
+        Response storedSourceBulkResponse = bulkIndex(client, "my-logs-ds-test", indexedWithStoredSource, 0);
+        assertOK(storedSourceBulkResponse);
+        assertThat(entityAsMap(storedSourceBulkResponse).get("errors"), Matchers.equalTo(false));
+
+        assertOK(putComponentTemplate(client, "my-logs-migrated-source", SYNTHETIC_SOURCE_COMPONENT_TEMPLATE));
+        var rolloverResponse = rolloverDataStream(client, "my-logs-ds-test");
+        assertOK(rolloverResponse);
+        assertThat(entityAsMap(rolloverResponse).get("rolled_over"), is(true));
+
+        var finalSourceMode = (String) getSetting(
+            client,
+            getDataStreamBackingIndex(client, "my-logs-ds-test", 1),
+            "index.mapping.source.mode"
+        );
+        assertThat(finalSourceMode, equalTo("synthetic"));
+
+        Response syntheticSourceBulkResponse = bulkIndex(client, "my-logs-ds-test", indexedWithSyntheticSource, 10);
+        assertOK(syntheticSourceBulkResponse);
+        assertThat(entityAsMap(syntheticSourceBulkResponse).get("errors"), Matchers.equalTo(false));
+
+        var allDocs = Stream.concat(indexedWithStoredSource.stream(), indexedWithSyntheticSource.stream()).toList();
+
+        var sourceList = search(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(allDocs.size()), "my-logs-ds-test");
+        assertThat(sourceList.size(), equalTo(allDocs.size()));
+
+        for (int i = 0; i < sourceList.size(); i++) {
+            var expected = XContentHelper.convertToMap(BytesReference.bytes(allDocs.get(i)), false, XContentType.JSON).v2();
+            assertThat(sourceList.get(i), equalTo(expected));
+        }
+    }
+
+    public void testSwitchFromSyntheticToStoredSource() throws IOException {
+        assertOK(putComponentTemplate(client, "my-logs-mapping", MAPPING_COMPONENT_TEMPLATE));
+        assertOK(putComponentTemplate(client, "my-logs-original-source", SYNTHETIC_SOURCE_COMPONENT_TEMPLATE));
+
+        assertOK(putTemplate(client, "my-logs", INDEX_TEMPLATE));
+        assertOK(createDataStream(client, "my-logs-ds-test"));
+
+        var initialSourceMode = (String) getSetting(
+            client,
+            getDataStreamBackingIndex(client, "my-logs-ds-test", 0),
+            "index.mapping.source.mode"
+        );
+        assertThat(initialSourceMode, equalTo("synthetic"));
+        var initialIndexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "my-logs-ds-test", 0), "index.mode");
+        assertThat(initialIndexMode, equalTo("logsdb"));
+
+        var indexedWithSyntheticSource = new ArrayList<XContentBuilder>();
+        var indexedWithStoredSource = new ArrayList<XContentBuilder>();
+        for (int i = 0; i < 10; i++) {
+            indexedWithSyntheticSource.add(generateDoc());
+            indexedWithStoredSource.add(generateDoc());
+        }
+
+        Response syntheticSourceBulkResponse = bulkIndex(client, "my-logs-ds-test", indexedWithSyntheticSource, 0);
+        assertOK(syntheticSourceBulkResponse);
+        assertThat(entityAsMap(syntheticSourceBulkResponse).get("errors"), Matchers.equalTo(false));
+
+        assertOK(putComponentTemplate(client, "my-logs-migrated-source", STORED_SOURCE_COMPONENT_TEMPLATE));
+        var rolloverResponse = rolloverDataStream(client, "my-logs-ds-test");
+        assertOK(rolloverResponse);
+        assertThat(entityAsMap(rolloverResponse).get("rolled_over"), is(true));
+
+        var finalSourceMode = (String) getSetting(
+            client,
+            getDataStreamBackingIndex(client, "my-logs-ds-test", 1),
+            "index.mapping.source.mode"
+        );
+        assertThat(finalSourceMode, equalTo("stored"));
+
+        Response storedSourceBulkResponse = bulkIndex(client, "my-logs-ds-test", indexedWithStoredSource, 10);
+        assertOK(storedSourceBulkResponse);
+        assertThat(entityAsMap(storedSourceBulkResponse).get("errors"), Matchers.equalTo(false));
+
+        var allDocs = Stream.concat(indexedWithSyntheticSource.stream(), indexedWithStoredSource.stream()).toList();
+
+        var sourceList = search(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(allDocs.size()), "my-logs-ds-test");
+        assertThat(sourceList.size(), equalTo(allDocs.size()));
+
+        for (int i = 0; i < sourceList.size(); i++) {
+            var expected = XContentHelper.convertToMap(BytesReference.bytes(allDocs.get(i)), false, XContentType.JSON).v2();
+            assertThat(sourceList.get(i), equalTo(expected));
+        }
+    }
+
+    private static Response bulkIndex(RestClient client, String dataStreamName, List<XContentBuilder> documents, int startId)
+        throws IOException {
+        var sb = new StringBuilder();
+        int id = startId;
+        for (var document : documents) {
+            sb.append(Strings.format("{ \"create\": { \"_id\" : \"%d\" } }", id)).append("\n");
+            sb.append(Strings.toString(document)).append("\n");
+            id++;
+        }
+
+        var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk");
+        bulkRequest.setJsonEntity(sb.toString());
+        bulkRequest.addParameter("refresh", "true");
+        return client.performRequest(bulkRequest);
+    }
+
+    @SuppressWarnings("unchecked")
+    private List<Map<String, Object>> search(SearchSourceBuilder search, String dataStreamName) throws IOException {
+        var request = new Request("GET", "/" + dataStreamName + "/_search");
+        request.setJsonEntity(Strings.toString(search));
+        var searchResponse = client.performRequest(request);
+        assertOK(searchResponse);
+
+        Map<String, Object> searchResponseMap = XContentHelper.convertToMap(
+            XContentType.JSON.xContent(),
+            searchResponse.getEntity().getContent(),
+            false
+        );
+        var hitsMap = (Map<String, Object>) searchResponseMap.get("hits");
+
+        var hitsList = (List<Map<String, Object>>) hitsMap.get("hits");
+        assertThat(hitsList.size(), greaterThan(0));
+
+        return hitsList.stream()
+            .sorted(Comparator.comparingInt((Map<String, Object> hit) -> Integer.parseInt((String) hit.get("_id"))))
+            .map(hit -> (Map<String, Object>) hit.get("_source"))
+            .toList();
+    }
+
+    private static XContentBuilder generateDoc() throws IOException {
+        var doc = XContentFactory.jsonBuilder();
+        doc.startObject();
+        {
+            doc.field("@timestamp", Long.toString(randomMillisUpToYear9999()));
+            doc.field("message", randomAlphaOfLengthBetween(20, 50));
+            doc.field("method", randomAlphaOfLength(3));
+            doc.field("hits", randomLong());
+        }
+        doc.endObject();
+
+        return doc;
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java
similarity index 83%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java
index db6c12c8bc565..f529b9fa1db96 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java
@@ -1,21 +1,17 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb;
+package org.elasticsearch.xpack.logsdb;
 
-import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.ClassRule;
 
@@ -115,18 +111,62 @@ public void testConfigureStoredSourceBeforeIndexCreation() throws IOException {
               }
             }""";
 
+        assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping));
+        assertOK(createDataStream(client, "logs-custom-dev"));
+
+        var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0));
+        String sourceMode = (String) subObject("_source").apply(mapping).get("mode");
+        assertThat(sourceMode, equalTo("stored"));
+    }
+
+    public void testConfigureDisabledSourceBeforeIndexCreation() {
+        var storedSourceMapping = """
+            {
+              "template": {
+                "settings": {
+                  "index": {
+                    "mode": "logsdb"
+                  }
+                },
+                "mappings": {
+                  "_source": {
+                    "enabled": false
+                  }
+                }
+              }
+            }""";
+
         Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping));
         assertThat(
             e.getMessage(),
-            containsString("Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source")
+            containsString("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode")
         );
         assertThat(e.getMessage(), containsString("mapper_parsing_exception"));
+    }
 
-        assertOK(createDataStream(client, "logs-custom-dev"));
+    public void testConfigureDisabledSourceModeBeforeIndexCreation() {
+        var storedSourceMapping = """
+            {
+              "template": {
+                "settings": {
+                  "index": {
+                    "mode": "logsdb"
+                  }
+                },
+                "mappings": {
+                  "_source": {
+                    "mode": "disabled"
+                  }
+                }
+              }
+            }""";
 
-        var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0));
-        String sourceMode = (String) subObject("_source").apply(mapping).get("mode");
-        assertThat(sourceMode, equalTo("synthetic"));
+        Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping));
+        assertThat(
+            e.getMessage(),
+            containsString("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode")
+        );
+        assertThat(e.getMessage(), containsString("mapper_parsing_exception"));
     }
 
     public void testConfigureStoredSourceWhenIndexIsCreated() throws IOException {
@@ -142,8 +182,45 @@ public void testConfigureStoredSourceWhenIndexIsCreated() throws IOException {
             }""";
 
         assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping));
+        assertOK(createDataStream(client, "logs-custom-dev"));
+
+        var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0));
+        String sourceMode = (String) subObject("_source").apply(mapping).get("mode");
+        assertThat(sourceMode, equalTo("stored"));
+    }
+
+    public void testConfigureDisabledSourceWhenIndexIsCreated() throws IOException {
+        var disabledModeMapping = """
+            {
+              "template": {
+                "mappings": {
+                  "_source": {
+                    "enabled": false
+                  }
+                }
+              }
+            }""";
+
+        assertOK(putComponentTemplate(client, "logs@custom", disabledModeMapping));
         ResponseException e = expectThrows(ResponseException.class, () -> createDataStream(client, "logs-custom-dev"));
-        assertThat(e.getMessage(), containsString("Indices with with index mode [logsdb] only support synthetic source"));
+        assertThat(e.getMessage(), containsString("_source can not be disabled in index using [logsdb] index mode"));
+    }
+
+    public void testConfigureDisabledSourceModeWhenIndexIsCreated() throws IOException {
+        var disabledModeMapping = """
+            {
+              "template": {
+                "mappings": {
+                  "_source": {
+                    "mode": "disabled"
+                  }
+                }
+              }
+            }""";
+
+        assertOK(putComponentTemplate(client, "logs@custom", disabledModeMapping));
+        ResponseException e = expectThrows(ResponseException.class, () -> createDataStream(client, "logs-custom-dev"));
+        assertThat(e.getMessage(), containsString("_source can not be disabled in index using [logsdb] index mode"));
     }
 
     public void testOverrideIndexCodec() throws IOException {
@@ -377,7 +454,7 @@ public void testIgnoreAboveSetting() throws IOException {
                 );
                 assertThat(
                     ex.getMessage(),
-                    Matchers.containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]")
+                    containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]")
                 );
             }
         }
@@ -418,16 +495,6 @@ public void testIgnoreAboveSetting() throws IOException {
         }
     }
 
-    private static Map<String, Object> getMapping(final RestClient client, final String indexName) throws IOException {
-        final Request request = new Request("GET", "/" + indexName + "/_mapping");
-
-        Map<String, Object> mappings = ((Map<String, Map<String, Object>>) entityAsMap(client.performRequest(request)).get(indexName)).get(
-            "mappings"
-        );
-
-        return mappings;
-    }
-
     private Function<Object, Map<String, Object>> subObject(String key) {
         return (mapAsObject) -> (Map<String, Object>) ((Map<String, Object>) mapAsObject).get(key);
     }
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeDisabledRestTestIT.java
similarity index 90%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeDisabledRestTestIT.java
index 123ca3b806153..40aab696dc9c4 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeDisabledRestTestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb;
+package org.elasticsearch.xpack.logsdb;
 
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.index.IndexMode;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java
similarity index 96%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java
index a024a2c0f303c..63094852c3626 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb;
+package org.elasticsearch.xpack.logsdb;
 
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java
similarity index 88%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java
index 22ac2b6d7d239..cc7f5bdb33871 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb;
+package org.elasticsearch.xpack.logsdb;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
@@ -100,4 +98,15 @@ protected static Response putClusterSetting(final RestClient client, final Strin
         request.setJsonEntity("{ \"transient\": { \"" + settingName + "\": " + settingValue + " } }");
         return client.performRequest(request);
     }
+
+    @SuppressWarnings("unchecked")
+    protected static Map<String, Object> getMapping(final RestClient client, final String indexName) throws IOException {
+        final Request request = new Request("GET", "/" + indexName + "/_mapping");
+
+        Map<String, Object> mappings = ((Map<String, Map<String, Object>>) entityAsMap(client.performRequest(request)).get(indexName)).get(
+            "mappings"
+        );
+
+        return mappings;
+    }
 }
diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
index b2d2978a254df..2bf8b00cf551c 100644
--- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.logsdb;
 
+import org.elasticsearch.client.Request;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -35,12 +37,6 @@ protected String getTestRestCluster() {
     }
 
     public void testFeatureUsageWithLogsdbIndex() throws IOException {
-        {
-            var response = getAsMap("/_license/feature_usage");
-            @SuppressWarnings("unchecked")
-            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
-            assertThat(features, Matchers.empty());
-        }
         {
             if (randomBoolean()) {
                 createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
@@ -75,7 +71,41 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             Map<?, ?> feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get();
             assertThat(feature.get("name"), equalTo("synthetic-source"));
             assertThat(feature.get("license_level"), equalTo("enterprise"));
+
+            var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings("test-index").get("test-index")).get("settings");
+            assertNull(settings.get("index.mapping.source.mode"));  // Default, no downgrading.
         }
     }
 
+    public void testLogsdbSourceModeForLogsIndex() throws IOException {
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }");
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/_index_template/1");
+        request.setJsonEntity("""
+            {
+                "index_patterns": ["logs-test-*"],
+                "data_stream": {
+                }
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/logs-test-foo/_doc");
+        request.setJsonEntity("""
+            {
+                "@timestamp": "2020-01-01T00:00:00.000Z",
+                "host.name": "foo",
+                "message": "bar"
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        String index = DataStream.getDefaultBackingIndexName("logs-test-foo", 1);
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertNull(settings.get("index.mapping.source.mode"));
+    }
+
 }
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/AbstractChallengeRestTest.java
similarity index 96%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/AbstractChallengeRestTest.java
index 6464b4e966823..60c7d07115ef2 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/AbstractChallengeRestTest.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java
new file mode 100644
index 0000000000000..c03e8aea9c2ac
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb.qa;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
+import org.elasticsearch.logsdb.datageneration.DocumentGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.Mapping;
+import org.elasticsearch.logsdb.datageneration.MappingGenerator;
+import org.elasticsearch.logsdb.datageneration.Template;
+import org.elasticsearch.logsdb.datageneration.TemplateGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.PredefinedField;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+public class DataGenerationHelper {
+    private final boolean keepArraySource;
+
+    private final DocumentGenerator documentGenerator;
+
+    private final Template template;
+    private final Mapping mapping;
+
+    public DataGenerationHelper() {
+        this(b -> {});
+    }
+
+    public DataGenerationHelper(Consumer<DataGeneratorSpecification.Builder> builderConfigurator) {
+        this.keepArraySource = ESTestCase.randomBoolean();
+
+        var specificationBuilder = DataGeneratorSpecification.builder()
+            .withFullyDynamicMapping(ESTestCase.randomBoolean())
+            .withPredefinedFields(
+                List.of(
+                    // Customized because it always needs doc_values for aggregations.
+                    new PredefinedField.WithGenerator(
+                        "host.name",
+                        FieldType.KEYWORD,
+                        Map.of("type", "keyword"),
+                        () -> ESTestCase.randomAlphaOfLength(5)
+                    ),
+                    // Needed for terms query
+                    new PredefinedField.WithGenerator(
+                        "method",
+                        FieldType.KEYWORD,
+                        Map.of("type", "keyword"),
+                        () -> ESTestCase.randomFrom("put", "post", "get")
+                    ),
+
+                    // Needed for histogram aggregation
+                    new PredefinedField.WithGenerator(
+                        "memory_usage_bytes",
+                        FieldType.LONG,
+                        Map.of("type", "long"),
+                        () -> ESTestCase.randomLongBetween(1000, 2000)
+                    )
+                )
+            );
+
+        // Customize builder if necessary
+        builderConfigurator.accept(specificationBuilder);
+
+        var specification = specificationBuilder.build();
+
+        this.documentGenerator = new DocumentGenerator(specification);
+
+        this.template = new TemplateGenerator(specification).generate();
+        this.mapping = new MappingGenerator(specification).generate(template);
+    }
+
+    void logsDbMapping(XContentBuilder builder) throws IOException {
+        builder.map(mapping.raw());
+    }
+
+    void standardMapping(XContentBuilder builder) throws IOException {
+        builder.map(mapping.raw());
+    }
+
+    void logsDbSettings(Settings.Builder builder) {
+        if (keepArraySource) {
+            builder.put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), "arrays");
+        }
+    }
+
+    void generateDocument(XContentBuilder document, Map<String, Object> additionalFields) throws IOException {
+        var generated = documentGenerator.generate(template, mapping);
+        generated.putAll(additionalFields);
+
+        document.map(generated);
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java
similarity index 84%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java
index e1cafc40f706f..0329f7723a108 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java
similarity index 84%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java
index dd80917b5f080..1c425cf30907b 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/ReindexChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/ReindexChallengeRestIT.java
similarity index 76%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/ReindexChallengeRestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/ReindexChallengeRestIT.java
index b48dce9ca4c57..83344b688ff8c 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/ReindexChallengeRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/ReindexChallengeRestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
similarity index 94%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
index 43efdbdcf8b1c..8d7a813b206d8 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
@@ -18,8 +16,6 @@
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.Matcher;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -31,6 +27,8 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.logsdb.qa.matchers.MatchResult;
+import org.elasticsearch.xpack.logsdb.qa.matchers.Matcher;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -53,7 +51,7 @@
  * This test uses simple mapping and document structure in order to allow easier debugging of the test itself.
  */
 public class StandardVersusLogsIndexModeChallengeRestIT extends AbstractChallengeRestTest {
-    private final int numShards = randomBoolean() ? randomIntBetween(2, 5) : 0;
+    private final int numShards = randomBoolean() ? randomIntBetween(2, 4) : 0;
     private final int numReplicas = randomBoolean() ? randomIntBetween(1, 3) : 0;
     private final boolean fullyDynamicMapping = randomBoolean();
 
@@ -303,6 +301,21 @@ public void testEsqlTermsAggregation() throws IOException {
         assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
+    public void testEsqlTermsAggregationByMethod() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
+
+        indexDocuments(documents);
+
+        final String query = "FROM $index | STATS count(*) BY method | SORT method | LIMIT " + numberOfDocuments;
+        final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings())
+            .settings(getContenderSettings(), getBaselineSettings())
+            .expected(getEsqlStatsResults(esqlBaseline(query)))
+            .ignoringSort(true)
+            .isEqualTo(getEsqlStatsResults(esqlContender(query)));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
+    }
+
     public void testFieldCaps() throws IOException {
         int numberOfDocuments = ESTestCase.randomIntBetween(20, 50);
         final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java
similarity index 72%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java
index 751336cc1f646..3b141908f45b1 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
@@ -17,6 +15,7 @@
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.Map;
 
 /**
  * Challenge test (see {@link StandardVersusLogsIndexModeChallengeRestIT}) that uses randomly generated
@@ -53,10 +52,10 @@ public void contenderSettings(Settings.Builder builder) {
     @Override
     protected XContentBuilder generateDocument(final Instant timestamp) throws IOException {
         var document = XContentFactory.jsonBuilder();
-        dataGenerationHelper.getDataGenerator().generateDocument(document, doc -> {
-            doc.field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp));
-        });
-
+        dataGenerationHelper.generateDocument(
+            document,
+            Map.of("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp))
+        );
         return document;
     }
 }
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT.java
similarity index 63%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT.java
index 6b0e4d4d0b34d..c1f97823b963a 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.common.settings.Settings;
 
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java
similarity index 74%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java
index d6cfebed1445a..5adf44f10be45 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa;
+package org.elasticsearch.xpack.logsdb.qa;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ArrayEqualMatcher.java
similarity index 83%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ArrayEqualMatcher.java
index b8f68a3c17494..b98ad65ac4d4f 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ArrayEqualMatcher.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
+package org.elasticsearch.xpack.logsdb.qa.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -15,8 +13,8 @@
 import java.util.Arrays;
 import java.util.List;
 
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintArrays;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintArrays;
 
 class ArrayEqualMatcher extends GenericEqualsMatcher<Object[]> {
     ArrayEqualMatcher(
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/GenericEqualsMatcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/GenericEqualsMatcher.java
similarity index 85%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/GenericEqualsMatcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/GenericEqualsMatcher.java
index 3edc9aeec6da8..933c7eb86f65a 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/GenericEqualsMatcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/GenericEqualsMatcher.java
@@ -1,20 +1,18 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
+package org.elasticsearch.xpack.logsdb.qa.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.util.List;
 
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
 
 public class GenericEqualsMatcher<T> extends Matcher {
     protected final XContentBuilder actualMappings;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ListEqualMatcher.java
similarity index 83%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ListEqualMatcher.java
index 00e22e9714283..447aa21b932c2 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ListEqualMatcher.java
@@ -1,21 +1,19 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
+package org.elasticsearch.xpack.logsdb.qa.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.util.List;
 
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintCollections;
 
 public class ListEqualMatcher extends GenericEqualsMatcher<List<?>> {
     public ListEqualMatcher(
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/MatchResult.java
similarity index 73%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/MatchResult.java
index 4334209bcfa30..a890a0375ef03 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/MatchResult.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
+package org.elasticsearch.xpack.logsdb.qa.matchers;
 
 import java.util.Objects;
 
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/Matcher.java
similarity index 90%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/Matcher.java
index 313bf3b3d2392..e08e401c19530 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/Matcher.java
@@ -1,17 +1,15 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
+package org.elasticsearch.xpack.logsdb.qa.matchers;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.source.SourceMatcher;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.logsdb.qa.matchers.source.SourceMatcher;
 
 import java.util.List;
 import java.util.Map;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Messages.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/Messages.java
similarity index 80%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Messages.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/Messages.java
index ddd8212c093eb..122e3b2d6261c 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Messages.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/Messages.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
+package org.elasticsearch.xpack.logsdb.qa.matchers;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ObjectMatcher.java
similarity index 68%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ObjectMatcher.java
index d071419f5aa6a..f2f08b1dfac14 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/ObjectMatcher.java
@@ -1,18 +1,16 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
+package org.elasticsearch.xpack.logsdb.qa.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
 
 public class ObjectMatcher extends GenericEqualsMatcher<Object> {
     ObjectMatcher(
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/DynamicFieldMatcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/DynamicFieldMatcher.java
similarity index 80%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/DynamicFieldMatcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/DynamicFieldMatcher.java
index 5d7ab12f06d7b..d6812c41f7611 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/DynamicFieldMatcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/DynamicFieldMatcher.java
@@ -1,17 +1,15 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+package org.elasticsearch.xpack.logsdb.qa.matchers.source;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.logsdb.qa.matchers.MatchResult;
 
 import java.util.List;
 import java.util.Objects;
@@ -20,8 +18,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintCollections;
 
 class DynamicFieldMatcher {
     private final XContentBuilder actualMappings;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/FieldSpecificMatcher.java
similarity index 92%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/FieldSpecificMatcher.java
index 96423125410a7..0c970f1b5fd9a 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/FieldSpecificMatcher.java
@@ -1,18 +1,16 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+package org.elasticsearch.xpack.logsdb.qa.matchers.source;
 
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.logsdb.qa.matchers.MatchResult;
 
 import java.math.BigInteger;
 import java.util.List;
@@ -22,8 +20,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintCollections;
 
 interface FieldSpecificMatcher {
     MatchResult match(List<Object> actual, List<Object> expected, Map<String, Object> actualMapping, Map<String, Object> expectedMapping);
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/MappingTransforms.java
similarity index 88%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/MappingTransforms.java
index ef5775eec3703..dbe73e3c2a4c2 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/MappingTransforms.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+package org.elasticsearch.xpack.logsdb.qa.matchers.source;
 
 import java.util.ArrayList;
 import java.util.HashMap;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/SourceMatcher.java
similarity index 90%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/SourceMatcher.java
index 0d12fabe081fa..cd2bb361d065d 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/SourceMatcher.java
@@ -1,29 +1,27 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+package org.elasticsearch.xpack.logsdb.qa.matchers.source;
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.GenericEqualsMatcher;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.ListEqualMatcher;
-import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.logsdb.qa.matchers.GenericEqualsMatcher;
+import org.elasticsearch.xpack.logsdb.qa.matchers.ListEqualMatcher;
+import org.elasticsearch.xpack.logsdb.qa.matchers.MatchResult;
 
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintCollections;
 
 public class SourceMatcher extends GenericEqualsMatcher<List<Map<String, Object>>> {
     private final Map<String, MappingTransforms.FieldMapping> actualNormalizedMapping;
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceTransforms.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/SourceTransforms.java
similarity index 86%
rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceTransforms.java
rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/SourceTransforms.java
index 23e3f090cafc3..c21383d411212 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceTransforms.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/matchers/source/SourceTransforms.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+package org.elasticsearch.xpack.logsdb.qa.matchers.source;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
index 49a83335671cd..089be0604146f 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
@@ -52,7 +52,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
             return List.of(logsdbIndexModeSettingsProvider);
         }
         return List.of(
-            new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
+            new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory(), logsdbIndexModeSettingsProvider),
             logsdbIndexModeSettingsProvider
         );
     }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
index b463426de0848..329cd3bc8a04b 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
@@ -42,25 +42,22 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) {
     public Settings getAdditionalIndexSettings(
         final String indexName,
         final String dataStreamName,
-        boolean isTimeSeries,
+        IndexMode templateIndexMode,
         final Metadata metadata,
         final Instant resolvedAt,
         final Settings settings,
         final List<CompressedXContent> combinedTemplateMappings
     ) {
-        if (isLogsdbEnabled == false || dataStreamName == null) {
-            return Settings.EMPTY;
-        }
-
-        final IndexMode indexMode = resolveIndexMode(settings.get(IndexSettings.MODE.getKey()));
-        if (indexMode != null) {
-            return Settings.EMPTY;
-        }
+        return getLogsdbModeSetting(dataStreamName, settings);
+    }
 
-        if (matchesLogsPattern(dataStreamName)) {
+    Settings getLogsdbModeSetting(final String dataStreamName, final Settings settings) {
+        if (isLogsdbEnabled
+            && dataStreamName != null
+            && resolveIndexMode(settings.get(IndexSettings.MODE.getKey())) == null
+            && matchesLogsPattern(dataStreamName)) {
             return Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build();
         }
-
         return Settings.EMPTY;
     }
 
@@ -71,5 +68,4 @@ private static boolean matchesLogsPattern(final String name) {
     private IndexMode resolveIndexMode(final String mode) {
         return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null;
     }
-
 }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
index 759fa6af98868..e7572d6a646e1 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -37,39 +38,58 @@ final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider
 
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
     private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
+    private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider;
 
     SyntheticSourceIndexSettingsProvider(
         SyntheticSourceLicenseService syntheticSourceLicenseService,
-        CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
+        CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory,
+        LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider
     ) {
         this.syntheticSourceLicenseService = syntheticSourceLicenseService;
         this.mapperServiceFactory = mapperServiceFactory;
+        this.logsdbIndexModeSettingsProvider = logsdbIndexModeSettingsProvider;
+    }
+
+    @Override
+    public boolean overrulesTemplateAndRequestSettings() {
+        // Indicates that the provider value takes precedence over any user setting.
+        return true;
     }
 
     @Override
     public Settings getAdditionalIndexSettings(
         String indexName,
         String dataStreamName,
-        boolean isTimeSeries,
+        IndexMode templateIndexMode,
         Metadata metadata,
         Instant resolvedAt,
         Settings indexTemplateAndCreateRequestSettings,
         List<CompressedXContent> combinedTemplateMappings
     ) {
+        var logsdbSettings = logsdbIndexModeSettingsProvider.getLogsdbModeSetting(dataStreamName, indexTemplateAndCreateRequestSettings);
+        if (logsdbSettings != Settings.EMPTY) {
+            indexTemplateAndCreateRequestSettings = Settings.builder()
+                .put(logsdbSettings)
+                .put(indexTemplateAndCreateRequestSettings)
+                .build();
+        }
+
         // This index name is used when validating component and index templates, we should skip this check in that case.
         // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
         boolean isTemplateValidation = "validate-index-name".equals(indexName);
-        if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
+        if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
             && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
             LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
-            // TODO: handle falling back to stored source
+            return Settings.builder()
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString())
+                .build();
         }
         return Settings.EMPTY;
     }
 
     boolean newIndexHasSyntheticSourceUsage(
         String indexName,
-        boolean isTimeSeries,
+        IndexMode templateIndexMode,
         Settings indexTemplateAndCreateRequestSettings,
         List<CompressedXContent> combinedTemplateMappings
     ) {
@@ -79,15 +99,17 @@ boolean newIndexHasSyntheticSourceUsage(
             return false;
         }
 
-        var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings);
-        try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
-            // combinedTemplateMappings can be null when creating system indices
-            // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
-            if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
-                combinedTemplateMappings = List.of(new CompressedXContent("{}"));
+        try {
+            var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings);
+            try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
+                // combinedTemplateMappings can be null when creating system indices
+                // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
+                if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
+                    combinedTemplateMappings = List.of(new CompressedXContent("{}"));
+                }
+                mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
+                return mapperService.documentMapper().sourceMapper().isSynthetic();
             }
-            mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
-            return mapperService.documentMapper().sourceMapper().isSynthetic();
         } catch (AssertionError | Exception e) {
             // In case invalid mappings or setting are provided, then mapper service creation can fail.
             // In that case it is ok to return false here. The index creation will fail anyway later, so need to fallback to stored source.
@@ -99,7 +121,7 @@ boolean newIndexHasSyntheticSourceUsage(
     // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
     private IndexMetadata buildIndexMetadataForMapperService(
         String indexName,
-        boolean isTimeSeries,
+        IndexMode templateIndexMode,
         Settings indexTemplateAndCreateRequestSettings
     ) {
         var tmpIndexMetadata = IndexMetadata.builder(indexName);
@@ -117,7 +139,7 @@ private IndexMetadata buildIndexMetadataForMapperService(
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
             .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
 
-        if (isTimeSeries) {
+        if (templateIndexMode == IndexMode.TIME_SERIES) {
             finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
             // Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
             finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
index 04e89af254f64..5f23dbdca1143 100644
--- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
@@ -51,7 +51,7 @@ public void testLogsDbDisabled() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             Metadata.EMPTY_METADATA,
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -69,7 +69,7 @@ public void testOnIndexCreation() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             "logs-apache-production",
             null,
-            false,
+            null,
             Metadata.EMPTY_METADATA,
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -87,7 +87,7 @@ public void testOnExplicitStandardIndex() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             Metadata.EMPTY_METADATA,
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(),
@@ -105,7 +105,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             Metadata.EMPTY_METADATA,
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(),
@@ -123,7 +123,7 @@ public void testNonLogsDataStream() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs",
-            false,
+            null,
             Metadata.EMPTY_METADATA,
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -141,7 +141,7 @@ public void testWithoutLogsComponentTemplate() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("*"), List.of()),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -159,7 +159,7 @@ public void testWithLogsComponentTemplate() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("*"), List.of("logs@settings")),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -177,7 +177,7 @@ public void testWithMultipleComponentTemplates() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -195,7 +195,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -213,7 +213,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -231,7 +231,7 @@ public void testCaseSensitivity() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "LOGS-apache-production",
-            false,
+            null,
             Metadata.EMPTY_METADATA,
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -249,7 +249,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException {
         final Settings additionalIndexSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production-eu",
-            false,
+            null,
             Metadata.EMPTY_METADATA,
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -267,7 +267,7 @@ public void testBeforeAndAFterSettingUpdate() throws IOException {
         final Settings beforeSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("*"), List.of("logs@settings")),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -281,7 +281,7 @@ public void testBeforeAndAFterSettingUpdate() throws IOException {
         final Settings afterSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("*"), List.of("logs@settings")),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
@@ -295,7 +295,7 @@ public void testBeforeAndAFterSettingUpdate() throws IOException {
         final Settings laterSettings = provider.getAdditionalIndexSettings(
             null,
             "logs-apache-production",
-            false,
+            null,
             buildMetadata(List.of("*"), List.of("logs@settings")),
             Instant.now().truncatedTo(ChronoUnit.SECONDS),
             Settings.EMPTY,
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
index c97328da132bd..2ab77b38b3373 100644
--- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
@@ -8,25 +8,51 @@
 package org.elasticsearch.xpack.logsdb;
 
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.List;
 
+import static org.elasticsearch.common.settings.Settings.builder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class SyntheticSourceIndexSettingsProviderTests extends ESTestCase {
 
+    private SyntheticSourceLicenseService syntheticSourceLicenseService;
     private SyntheticSourceIndexSettingsProvider provider;
 
+    private static LogsdbIndexModeSettingsProvider getLogsdbIndexModeSettingsProvider(boolean enabled) {
+        return new LogsdbIndexModeSettingsProvider(Settings.builder().put("cluster.logsdb.enabled", enabled).build());
+    }
+
     @Before
     public void setup() {
-        SyntheticSourceLicenseService syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY);
+        MockLicenseState licenseState = mock(MockLicenseState.class);
+        when(licenseState.isAllowed(any())).thenReturn(true);
+        var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY);
+        licenseService.setLicenseState(licenseState);
+        syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY);
+        syntheticSourceLicenseService.setLicenseState(licenseState);
+
         provider = new SyntheticSourceIndexSettingsProvider(
             syntheticSourceLicenseService,
-            im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName())
+            im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
+            getLogsdbIndexModeSettingsProvider(false)
         );
     }
 
@@ -49,7 +75,7 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException {
                     }
                 }
                 """;
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
             assertTrue(result);
         }
         {
@@ -82,7 +108,7 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException {
                     }
                     """;
             }
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
             assertFalse(result);
         }
     }
@@ -104,7 +130,7 @@ public void testValidateIndexName() throws IOException {
             }
             """;
         Settings settings = Settings.EMPTY;
-        boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+        boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
         assertFalse(result);
     }
 
@@ -124,22 +150,22 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException
             """;
         {
             Settings settings = Settings.builder().put("index.mode", "logsdb").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
             assertTrue(result);
         }
         {
             Settings settings = Settings.builder().put("index.mode", "logsdb").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of());
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of());
             assertTrue(result);
         }
         {
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, Settings.EMPTY, List.of());
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of());
             assertFalse(result);
         }
         {
             boolean result = provider.newIndexHasSyntheticSourceUsage(
                 indexName,
-                false,
+                null,
                 Settings.EMPTY,
                 List.of(new CompressedXContent(mapping))
             );
@@ -164,22 +190,22 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException {
             """;
         {
             Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
             assertTrue(result);
         }
         {
             Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of());
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of());
             assertTrue(result);
         }
         {
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, Settings.EMPTY, List.of());
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of());
             assertFalse(result);
         }
         {
             boolean result = provider.newIndexHasSyntheticSourceUsage(
                 indexName,
-                false,
+                null,
                 Settings.EMPTY,
                 List.of(new CompressedXContent(mapping))
             );
@@ -206,7 +232,7 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep
                     }
                 }
                 """;
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
             assertFalse(result);
         }
         {
@@ -221,9 +247,140 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep
                     }
                 }
                 """;
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
             assertFalse(result);
         }
     }
 
+    public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException {
+        String dataStreamName = "logs-app1";
+        Metadata.Builder mb = Metadata.builder(
+            DataStreamTestHelper.getClusterStateWithDataStreams(
+                List.of(Tuple.tuple(dataStreamName, 1)),
+                List.of(),
+                Instant.now().toEpochMilli(),
+                builder().build(),
+                1
+            ).getMetadata()
+        );
+        Metadata metadata = mb.build();
+
+        Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+            .build();
+
+        Settings result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            null,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(0));
+
+        syntheticSourceLicenseService.setSyntheticSourceFallback(true);
+        result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            null,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(1));
+        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+
+        result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            IndexMode.TIME_SERIES,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(1));
+        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+
+        result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            IndexMode.LOGSDB,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(1));
+        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+    }
+
+    public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException {
+        syntheticSourceLicenseService.setSyntheticSourceFallback(true);
+        provider = new SyntheticSourceIndexSettingsProvider(
+            syntheticSourceLicenseService,
+            im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
+            getLogsdbIndexModeSettingsProvider(true)
+        );
+        final Settings settings = Settings.EMPTY;
+
+        String dataStreamName = "logs-app1";
+        Metadata.Builder mb = Metadata.builder(
+            DataStreamTestHelper.getClusterStateWithDataStreams(
+                List.of(Tuple.tuple(dataStreamName, 1)),
+                List.of(),
+                Instant.now().toEpochMilli(),
+                builder().build(),
+                1
+            ).getMetadata()
+        );
+        Metadata metadata = mb.build();
+        Settings result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            null,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(0));
+
+        dataStreamName = "logs-app1-0";
+        mb = Metadata.builder(
+            DataStreamTestHelper.getClusterStateWithDataStreams(
+                List.of(Tuple.tuple(dataStreamName, 1)),
+                List.of(),
+                Instant.now().toEpochMilli(),
+                builder().build(),
+                1
+            ).getMetadata()
+        );
+        metadata = mb.build();
+
+        result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            null,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(1));
+        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+
+        result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            null,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.toString()).build(),
+            List.of()
+        );
+        assertThat(result.size(), equalTo(0));
+    }
 }
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_failure_store.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_failure_store.yml
new file mode 100644
index 0000000000000..21e4f49fe7af5
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_failure_store.yml
@@ -0,0 +1,99 @@
+---
+teardown:
+  - do:
+      indices.delete_data_stream:
+        name: my-logs-fs
+        ignore: 404
+
+  - do:
+      indices.delete_index_template:
+        name: template
+        ignore: 404
+
+  - do:
+      indices.delete_data_stream:
+        name: my-logs-db
+        ignore: 404
+  - do:
+      indices.delete_index_template:
+        name: template1
+        ignore: 404
+
+---
+Test failure store with logsdb:
+  - requires:
+      test_runner_features: [ capabilities, allowed_warnings ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logsdb_index_mode ]
+        - method: POST
+          path: /_bulk
+          capabilities: [ 'failure_store_status' ]
+        - method: PUT
+          path: /_bulk
+          capabilities: [ 'failure_store_status' ]
+      reason: "Support for 'logsdb' index mode & failure status capability required"
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template] has index patterns [my-logs-fs*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["my-logs-fs*"]
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              index:
+                mode: logsdb
+                number_of_replicas: 1
+                number_of_shards: 2
+  - do:
+      allowed_warnings:
+        - "index template [my-template2] has index patterns [my-logs-db*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [ "my-logs-db*" ]
+          data_stream: {}
+          template:
+            settings:
+              index:
+                mode: logsdb
+                number_of_replicas: 1
+                number_of_shards: 2
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "create": { "_index": "my-logs-fs"} }'
+          - '{"@timestamp":"2019-08-06T12:09:12.375Z", "log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer"}'
+          - '{ "create": { "_index": "my-logs-db"} }'
+          - '{ "@timestamp": "2022-01-01", "log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer" }'
+          - '{ "create": { "_index": "my-logs-fs"} }'
+          - '{"log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer"}'
+          - '{ "create": { "_index": "my-logs-db"} }'
+          - '{"log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer"}'
+  - is_true: errors
+
+  # Successfully indexed to backing index
+  - match: { items.0.create._index: '/\.ds-my-logs-fs-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { items.0.create.status: 201 }
+  - is_false: items.0.create.failure_store
+  - match: { items.1.create._index: '/\.ds-my-logs-db-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { items.1.create.status: 201 }
+  - is_false: items.1.create.failure_store
+
+  # Successfully indexed to failure store
+  - match: { items.2.create._index: '/\.fs-my-logs-fs-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - match: { items.2.create.status: 201 }
+  - match: { items.2.create.failure_store: used }
+
+  # Rejected, eligible to go to failure store, but failure store not enabled
+  - match: { items.3.create._index: '/\.ds-my-logs-db-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { items.3.create.status: 400 }
+  - match: { items.3.create.error.type: document_parsing_exception }
+  - match: { items.3.create.failure_store: not_enabled }
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml
index 52c500c102cee..2f320c2cad966 100644
--- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml
+++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml
@@ -280,3 +280,784 @@ create logsdb data stream with timestamp object mapping:
 
   - match: { error.type: "illegal_argument_exception" }
   - match: { error.reason: "composable template [logsdb-index-template] template after composition with component templates [logsdb-mappings] is invalid" }
+
+---
+create logsdb data stream with custom sorting without host.name:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-http-prod] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-http-prod ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ agent.id ]
+                sort.order: [ desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                agent.id:
+                  type: keyword
+                host.hostname:
+                  type: keyword
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-http-prod
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logs-http-prod
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.agent.properties.id.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: null }
+
+---
+create logsdb data stream with custom sorting and host object:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-nginx-prod] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-nginx-prod ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ host.hostname, host.region ]
+                sort.order: [ desc, desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host:
+                  type: object
+                  properties:
+                    ip:
+                      type: ip
+                    hostname:
+                      type: keyword
+                    region:
+                      type: keyword
+                    name:
+                      type: integer
+
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-nginx-prod
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logs-nginx-prod
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.ip.type: ip }
+  - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.region.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: integer } # Overrides LogsDB injected
+
+---
+create logsdb data stream with custom sorting and dynamically mapped host.name:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-kafka-qa] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-kafka-qa ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ "agent.id", "@timestamp" ]
+                sort.order: [ desc, asc ]
+                mode: logsdb
+            mappings:
+              properties:
+                agent:
+                  type: object
+                  properties:
+                    name:
+                      type: keyword
+                    id:
+                      type: keyword
+
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-kafka-qa
+  - is_true: acknowledged
+
+  - do:
+      bulk:
+        index: logs-kafka-qa
+        refresh: true
+        body:
+          - { "create": { } }
+          - { "@timestamp": "2022-01-01T00:00:00", agent.name: "foo", agent.id: "foo-568", host: { id: "db8fdcf1-b1e2-444b-8c6a-0466c61dcce4" } }
+          - { "create": { } }
+          - { "@timestamp": "2022-01-01T00:01:00", agent.name: "bar", agent.id: "foo-309", host: { id: "35e1ed10-961e-46c7-83ea-4109c913a1d6" } }
+
+  - do:
+      indices.get_data_stream:
+        name: logs-kafka-qa
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.agent.properties.name.type: keyword }
+  - match: { .$backing_index.mappings.properties.agent.properties.id.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name: null }
+  - match: { .$backing_index.mappings.properties.host.properties.id.type: text }
+
+---
+create logsdb data stream with custom sorting and host.name object:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-nginx-qa] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-nginx-qa ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ "host.name.value", "@timestamp" ]
+                sort.order: [ desc, desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host:
+                  type: object
+                  properties:
+                    name:
+                      type: object
+                      properties:
+                        value:
+                          type: keyword
+                        alias:
+                          type: keyword
+
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-nginx-qa
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logs-nginx-qa
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.name.properties.value.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.properties.alias.type: keyword }
+
+---
+create logsdb data stream with default sorting on malformed host.name:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-win-prod] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-win-prod ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                mode: logsdb
+            mappings:
+              properties:
+                agent:
+                  type: object
+                  properties:
+                    name:
+                      type: keyword
+                    id:
+                      type: keyword
+
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-win-prod
+  - is_true: acknowledged
+
+  - do:
+      bulk:
+        index: logs-win-prod
+        refresh: true
+        body:
+          - { "create": { } }
+          - { "@timestamp": "2022-01-01T00:00:00", agent.name: "foo", agent.id: "foo-568", host: { name: 192.168.10.12, id: "e70e91cd-bb3f-43f0-909c-2748e7fdfd54" } }
+          - { "create": { } }
+          - { "@timestamp": "2022-01-01T00:01:00", agent.name: "bar", agent.id: "foo-309", host: { name: 192.168.15.17, id: "ad2e3edb-2c4b-4f12-83dd-255691ed614c" } }
+
+  - do:
+      indices.get_data_stream:
+        name: logs-win-prod
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.agent.properties.name.type: keyword }
+  - match: { .$backing_index.mappings.properties.agent.properties.id.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } # LogsDB injected
+  - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 } # LogsDB injected
+  - match: { .$backing_index.mappings.properties.host.properties.id.type: text }
+
+---
+create logsdb data stream with custom sorting and host.name date field:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-http-prod] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-http-prod ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ host.name, host.hostname ]
+                sort.order: [ desc, desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host:
+                  type: object
+                  properties:
+                    hostname:
+                      type: keyword
+                    name:
+                      type: date
+
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-http-prod
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logs-http-prod
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: date }
+
+---
+create logsdb data stream with custom sorting and missing host.name field mapping:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-http-qa] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-http-qa ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ host.name, host.hostname ]
+                sort.order: [ desc, desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host:
+                  type: object
+                  properties:
+                    hostname:
+                      type: keyword
+
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-http-qa
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logs-http-qa
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 }
+
+---
+create logsdb data stream with custom sorting and host.name field without doc values:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-http-dev] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-http-dev ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ "host.name", "@timestamp" ]
+                sort.order: [ desc, desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host:
+                  type: object
+                  properties:
+                    name:
+                      type: keyword
+                      doc_values: false
+
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      catch: bad_request
+      indices.create_data_stream:
+        name: logs-http-dev
+
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "docvalues not found for index sort field:[host.name]" }
+
+---
+create logsdb data stream with incompatible ignore_above on host.name:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logsdb-index-template-ignore-above] has index patterns [logsdb-ignore-above] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template-ignore-above] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logsdb-index-template-ignore-above
+        body:
+          index_patterns: [ logsdb-ignore-above ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ host.name ]
+                sort.order: [ desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host.name:
+                  type: keyword
+                  ignore_above: 128
+          data_stream: {}
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logsdb-ignore-above
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logsdb-ignore-above
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 128 }
+
+---
+create logsdb data stream with no sorting and host.name as text:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logsdb-index-template-non-keyword] has index patterns [logsdb-non-keyword] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template-non-keyword] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logsdb-index-template-non-keyword
+        body:
+          index_patterns: [ logsdb-non-keyword ]
+          priority: 10000
+          template:
+            settings:
+              mode: logsdb
+            mappings:
+              properties:
+                host.name:
+                  type: text
+          data_stream: {}
+  - is_true: acknowledged
+
+  - do:
+      catch: bad_request
+      indices.create_data_stream:
+        name: logsdb-non-keyword
+
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "docvalues not found for index sort field:[host.name]" }
+
+---
+create logsdb data stream without index sorting and ignore_above on host.name:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logsdb-index-template-ignore-above-override] has index patterns [logsdb-ignore-above-override] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template-ignore-above-override] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logsdb-index-template-ignore-above-override
+        body:
+          index_patterns: [ logsdb-ignore-above-override ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                mode: logsdb
+            mappings:
+              properties:
+                host.name:
+                  type: keyword
+                  ignore_above: 128
+          data_stream: {}
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logsdb-ignore-above-override
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logsdb-ignore-above-override
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 128 }
+
+---
+create logsdb data stream with host.name as alias and sorting on it:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logsdb-index-template-alias] has index patterns [logsdb-alias] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template-alias] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logsdb-index-template-alias
+        body:
+          index_patterns: [ logsdb-alias ]
+          template:
+            settings:
+              index:
+                sort.field: [ host.name ]
+                sort.order: [ desc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host.name:
+                  type: alias
+                  path: host.hostname
+                host.hostname:
+                  type:
+                    keyword
+          data_stream: {}
+  - do:
+      catch: bad_request
+      indices.create_data_stream:
+        name: logsdb-alias
+
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "Cannot use alias [host.name] as an index sort field" }
+
+---
+create logsdb data stream with multi-fields on host.name:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logsdb-index-template-multi-fields] has index patterns [logsdb-multi-fields] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template-multi-fields] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logsdb-index-template-multi-fields
+        body:
+          index_patterns: [ logsdb-multi-fields ]
+          template:
+            settings:
+              index:
+                sort.field: [ host.name.keyword ]
+                sort.order: [ asc ]
+                mode: logsdb
+            mappings:
+              properties:
+                host.name:
+                  type: "text"
+                  fields:
+                    keyword:
+                      type: "keyword"
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: logsdb-multi-fields
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logsdb-multi-fields
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.name.fields.keyword.type: keyword }
+
+---
+create logsdb data stream with multi-fields on host.name and no sorting:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logsdb-no-sort-multi-fields-template] has index patterns [logsdb-no-sort-multi-fields] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-no-sort-multi-fields-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logsdb-no-sort-multi-fields-template
+        body:
+          index_patterns: [ logsdb-no-sort-multi-fields ]
+          template:
+            settings:
+              mode: logsdb
+            mappings:
+              properties:
+                host.name:
+                  type: text
+                  fields:
+                    keyword:
+                      type: keyword
+          data_stream: {}
+
+  - do:
+      catch: bad_request
+      indices.create_data_stream:
+        name: logsdb-no-sort-multi-fields
+
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "docvalues not found for index sort field:[host.name]" }
+
+---
+create logsdb data stream with custom empty sorting:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-http-empty] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-http-empty ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ ]
+                sort.order: [ ]
+                mode: logsdb
+            mappings:
+              properties:
+                hostname:
+                  type: keyword
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-http-empty
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logs-http-empty
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword }
+  - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 }
+
+---
+create logsdb data stream with custom sorting on timestamp:
+  - skip:
+      features: [ "allowed_warnings" ]
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+
+  - do:
+      allowed_warnings:
+        - "index template [logs-template] has index patterns [logs-http-dev] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: logs-template
+        body:
+          index_patterns: [ logs-http-dev ]
+          priority: 10000
+          template:
+            settings:
+              index:
+                sort.field: [ "@timestamp" ]
+                sort.order: [ "asc" ]
+                mode: logsdb
+            mappings:
+              properties:
+                hostname:
+                  type: keyword
+          data_stream: { }
+  - is_true: acknowledged
+
+  - do:
+      indices.create_data_stream:
+        name: logs-http-dev
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_stream:
+        name: logs-http-dev
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml
new file mode 100644
index 0000000000000..33fedce3b59c1
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml
@@ -0,0 +1,847 @@
+---
+create an index with disabled source mode and standard index mode without setting:
+  - do:
+      indices.create:
+        index: test_disabled_standard
+        body:
+          settings:
+            index:
+              mode: standard
+          mappings:
+            _source:
+              mode: disabled
+
+  - do:
+      indices.get_mapping:
+        index: test_disabled_standard
+
+  - match: { test_disabled_standard.mappings._source.mode: disabled }
+
+---
+create an index with stored source mode and standard index mode without setting:
+  - do:
+      indices.create:
+        index: test_stored_standard
+        body:
+          settings:
+            index:
+              mode: standard
+          mappings:
+            _source:
+              mode: stored
+
+  - do:
+      indices.get_mapping:
+        index: test_stored_standard
+
+  - match: { test_stored_standard.mappings._source.mode: stored }
+
+---
+create an index with synthetic source mode and standard index mode without setting:
+  - do:
+      indices.create:
+        index: test_synthetic_standard
+        body:
+          settings:
+            index:
+              mode: standard
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      indices.get_mapping:
+        index: test_synthetic_standard
+
+  - match: { test_synthetic_standard.mappings._source.mode: synthetic }
+
+---
+create an index with disabled source mode and logsdb index mode without setting:
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test_disabled_logsdb
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              mode: disabled
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
+
+---
+create an index with stored source mode and logsdb index mode without setting:
+  - do:
+      indices.create:
+        index: test_stored_logsdb
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              mode: stored
+
+  - do:
+      indices.get_settings:
+        index: "test_stored_logsdb"
+  - match: { test_stored_logsdb.settings.index.mode: logsdb }
+
+  - do:
+      indices.get_mapping:
+        index: test_stored_logsdb
+
+  - match: { test_stored_logsdb.mappings._source.mode: stored }
+
+---
+create an index with synthetic source mode and logsdb index mode without setting:
+  - do:
+      indices.create:
+        index: test_synthetic_logsdb
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      indices.get_mapping:
+        index: test_synthetic_logsdb
+
+  - match: { test_synthetic_logsdb.mappings._source.mode: synthetic }
+
+---
+create an index with disabled source mode and time series index mode without setting:
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test_disabled_time_series
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              mode: disabled
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
+
+---
+create an index with stored source mode and time series index mode without setting:
+  - do:
+      indices.create:
+        index: test_stored_time_series
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              mode: stored
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      indices.get_settings:
+        index: "test_stored_time_series"
+  - match: { test_stored_time_series.settings.index.mode: time_series }
+
+  - do:
+      indices.get_mapping:
+        index: test_stored_time_series
+
+  - match: { test_stored_time_series.mappings._source.mode: stored }
+
+
+---
+create an index with synthetic source mode and time series index mode without setting:
+  - do:
+      indices.create:
+        index: test_synthetic_time_series
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      indices.get_settings:
+        index: "test_synthetic_time_series"
+  - match: { test_synthetic_time_series.settings.index.mode: time_series }
+
+  - do:
+      indices.get_mapping:
+        index: test_synthetic_time_series
+
+  - match: { test_synthetic_time_series.mappings._source.mode: synthetic }
+
+---
+create an index with stored source mode:
+  - do:
+      indices.create:
+        index: test_stored_default
+        body:
+          mappings:
+            _source:
+              mode: stored
+
+  - do:
+      indices.get_mapping:
+        index: test_stored_default
+
+  - match: { test_stored_default.mappings._source.mode: stored }
+
+---
+override stored to synthetic source mode:
+  - do:
+      indices.create:
+        index: test_stored_override
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+          mappings:
+            _source:
+              mode: stored
+
+  - do:
+      indices.get_mapping:
+        index: test_stored_override
+
+  - match: { test_stored_override.mappings._source.mode: synthetic }
+
+---
+override stored to disabled source mode:
+  - do:
+      indices.create:
+        index: test_stored_disabled
+        body:
+          settings:
+            index:
+              mapping.source.mode: disabled
+          mappings:
+            _source:
+              mode: stored
+
+  - do:
+      indices.get_mapping:
+        index: test_stored_disabled
+
+  - match: { test_stored_disabled.mappings._source.mode: disabled }
+
+---
+create an index with disabled source mode:
+  - do:
+      indices.create:
+        index: test_disabled_default
+        body:
+          mappings:
+            _source:
+              mode: disabled
+
+  - do:
+      indices.get_mapping:
+        index: test_disabled_default
+
+  - match: { test_disabled_default.mappings._source.mode: disabled }
+
+---
+override disabled to synthetic source mode:
+  - do:
+      indices.create:
+        index: test_disabled_synthetic
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+          mappings:
+            _source:
+              mode: disabled
+
+  - do:
+      indices.get_mapping:
+        index: test_disabled_synthetic
+
+  - match: { test_disabled_synthetic.mappings._source.mode: synthetic }
+
+---
+override disabled to stored source mode:
+  - do:
+      indices.create:
+        index: test_disabled_stored
+        body:
+          settings:
+            index:
+              mapping.source.mode: stored
+          mappings:
+            _source:
+              mode: disabled
+
+  - do:
+      indices.get_mapping:
+        index: test_disabled_stored
+
+  - match: { test_disabled_stored.mappings._source.mode: stored }
+
+---
+create an index with synthetic source mode:
+  - do:
+      indices.create:
+        index: test_synthetic_default
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      indices.get_mapping:
+        index: test_synthetic_default
+
+  - match: { test_synthetic_default.mappings._source.mode: synthetic }
+
+---
+override synthetic to stored source mode:
+  - do:
+      indices.create:
+        index: test_synthetic_stored
+        body:
+          settings:
+            index:
+              mapping.source.mode: stored
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      indices.get_mapping:
+        index: test_synthetic_stored
+
+  - match: { test_synthetic_stored.mappings._source.mode: stored }
+
+---
+override synthetic to disabled source mode:
+  - do:
+      indices.create:
+        index: test_synthetic_disabled
+        body:
+          settings:
+            index:
+              mapping.source.mode: disabled
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      indices.get_mapping:
+        index: test_synthetic_disabled
+
+  - match: { test_synthetic_disabled.mappings._source.mode: disabled }
+
+---
+create an index with unspecified source mode:
+  - do:
+      indices.create:
+        index: test_unset_default
+
+  - do:
+      indices.get_mapping:
+        index: test_unset_default
+
+  - match: { test_unset_default.mappings._source.mode: null }
+
+---
+override unspecified to stored source mode:
+  - do:
+      indices.create:
+        index: test_unset_stored
+        body:
+          settings:
+            index:
+              mapping.source.mode: stored
+
+  - do:
+      indices.get_mapping:
+        index: test_unset_stored
+
+  - match: { test_unset_stored.mappings: { } }
+
+---
+override unspecified to disabled source mode:
+  - do:
+      indices.create:
+        index: test_unset_disabled
+        body:
+          settings:
+            index:
+              mapping.source.mode: disabled
+
+  - do:
+      indices.get_mapping:
+        index: test_unset_disabled
+
+  - match: { test_unset_disabled.mappings: { } }
+
+---
+override unspecified to synthetic source mode:
+  - do:
+      indices.create:
+        index: test_unset_synthetic
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+
+  - do:
+      indices.get_mapping:
+        index: test_unset_synthetic
+
+  - match: { test_unset_synthetic.mappings: { } }
+
+---
+create an index with standard index mode:
+  - do:
+      indices.create:
+        index: test_standard_index_mode
+        body:
+          settings:
+            index:
+              mode: standard
+          mappings:
+            _source:
+              mode: stored
+
+  - do:
+      indices.get_mapping:
+        index: test_standard_index_mode
+
+  - match: { test_standard_index_mode.mappings._source.mode: stored }
+
+---
+create an index with time_series index mode and synthetic source:
+  - do:
+      indices.create:
+        index: test_time_series_index_mode_synthetic
+        body:
+          settings:
+            index:
+              mode: time_series
+              mapping.source.mode: synthetic
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      indices.get_settings:
+        index: "test_time_series_index_mode_synthetic"
+  - match: { test_time_series_index_mode_synthetic.settings.index.mode: time_series }
+
+
+  - do:
+      indices.get_mapping:
+        index: test_time_series_index_mode_synthetic
+
+  - match: { test_time_series_index_mode_synthetic.mappings._source.mode: synthetic }
+
+---
+create an index with logsdb index mode and synthetic source:
+  - do:
+      indices.create:
+        index: test_logsdb_index_mode_synthetic
+        body:
+          settings:
+            index:
+              mode: logsdb
+              mapping.source.mode: synthetic
+
+  - do:
+      indices.get_settings:
+        index: "test_logsdb_index_mode_synthetic"
+  - match: { test_logsdb_index_mode_synthetic.settings.index.mode: logsdb }
+
+  - do:
+      indices.get_mapping:
+        index: test_logsdb_index_mode_synthetic
+
+  - match: { test_logsdb_index_mode_synthetic.mappings._source.mode: synthetic }
+
+---
+create an index with time_series index mode and stored source:
+  - do:
+      indices.create:
+        index: test_time_series_index_mode_undefined
+        body:
+          settings:
+            index:
+              mode: time_series
+              mapping.source.mode: stored
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      indices.get_settings:
+        index: "test_time_series_index_mode_undefined"
+  - match: { test_time_series_index_mode_undefined.settings.index.mode: time_series }
+
+  - do:
+      indices.get_mapping:
+        index: test_time_series_index_mode_undefined
+
+  - match: { test_time_series_index_mode_undefined.mappings._source.mode: stored }
+
+---
+create an index with logsdb index mode and stored source:
+  - do:
+      indices.create:
+        index: test_logsdb_index_mode_undefined
+        body:
+          settings:
+            index:
+              mode: logsdb
+              mapping.source.mode: stored
+
+  - do:
+      indices.get_settings:
+        index: "test_logsdb_index_mode_undefined"
+  - match: { test_logsdb_index_mode_undefined.settings.index.mode: logsdb }
+
+  - do:
+      indices.get_mapping:
+        index: test_logsdb_index_mode_undefined
+
+  - match: { test_logsdb_index_mode_undefined.mappings._source.mode: stored }
+
+---
+create an index with time_series index mode and disabled source:
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test_time_series_index_mode
+        body:
+          settings:
+            index:
+              mode: time_series
+              mapping.source.mode: disabled
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
+
+---
+create an index with logsdb index mode and disabled source:
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test_logsdb_index_mode
+        body:
+          settings:
+            index:
+              mode: logsdb
+              mapping.source.mode: disabled
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
+
+---
+modify final setting after index creation:
+  - do:
+      indices.create:
+        index: test_modify_setting
+        body:
+          settings:
+            index:
+              mapping.source.mode: stored
+
+  - do:
+      catch: /.*Can't update non dynamic setting.*/
+      indices.put_settings:
+        index: test_modify_setting
+        body:
+          index:
+            mapping.source.mode: synthetic
+
+---
+modify source mapping from stored to disabled after index creation:
+  - do:
+      indices.create:
+        index: test_modify_source_mode_stored_disabled
+        body:
+          settings:
+            index:
+              mapping.source.mode: stored
+
+  - do:
+      indices.put_mapping:
+        index: test_modify_source_mode_stored_disabled
+        body:
+          _source:
+            mode: disabled
+  - is_true: acknowledged
+
+  - do:
+      indices.get_mapping:
+        index: test_modify_source_mode_stored_disabled
+  - match: { test_modify_source_mode_stored_disabled.mappings._source.mode: stored }
+
+---
+modify source mapping from stored to synthetic after index creation:
+  - do:
+      indices.create:
+        index: test_modify_source_mode_stored_synthetic
+        body:
+          settings:
+            index:
+              mapping.source.mode: stored
+
+  - do:
+      indices.put_mapping:
+        index: test_modify_source_mode_stored_synthetic
+        body:
+          _source:
+            mode: synthetic
+  - is_true: acknowledged
+
+  - do:
+      indices.get_mapping:
+        index: test_modify_source_mode_stored_synthetic
+  - match: { test_modify_source_mode_stored_synthetic.mappings._source.mode: stored }
+
+---
+modify source mapping from disabled to stored after index creation:
+  - do:
+      indices.create:
+        index: test_modify_source_mode_disabled_stored
+        body:
+          settings:
+            index:
+              mapping.source.mode: disabled
+
+  - do:
+      indices.put_mapping:
+        index: test_modify_source_mode_disabled_stored
+        body:
+          _source:
+            mode: stored
+  - is_true: acknowledged
+
+  - do:
+      indices.get_mapping:
+        index: test_modify_source_mode_disabled_stored
+  - match: { test_modify_source_mode_disabled_stored.mappings._source.mode: disabled }
+
+---
+modify source mapping from disabled to synthetic after index creation:
+  - do:
+      indices.create:
+        index: test_modify_source_mode_disabled_synthetic
+        body:
+          settings:
+            index:
+              mapping.source.mode: disabled
+
+  - do:
+      indices.put_mapping:
+        index: test_modify_source_mode_disabled_synthetic
+        body:
+          _source:
+            mode: synthetic
+  - is_true: acknowledged
+
+  - do:
+      indices.get_mapping:
+        index: test_modify_source_mode_disabled_synthetic
+  - match: { test_modify_source_mode_disabled_synthetic.mappings._source.mode: disabled }
+
+---
+modify source mapping from synthetic to stored after index creation:
+  - do:
+      indices.create:
+        index: test_modify_source_mode_synthetic_stored
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+
+  - do:
+      indices.put_mapping:
+        index: test_modify_source_mode_synthetic_stored
+        body:
+          _source:
+            mode: stored
+  - is_true: acknowledged
+
+  - do:
+      indices.get_mapping:
+        index: test_modify_source_mode_synthetic_stored
+  - match: { test_modify_source_mode_synthetic_stored.mappings._source.mode: synthetic }
+
+---
+modify source mapping from synthetic to disabled after index creation:
+  - do:
+      indices.create:
+        index: test_modify_source_mode_synthetic_disabled
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+
+  - do:
+      indices.put_mapping:
+        index: test_modify_source_mode_synthetic_disabled
+        body:
+          _source:
+            mode: disabled
+  - is_true: acknowledged
+
+  - do:
+      indices.get_mapping:
+        index: test_modify_source_mode_synthetic_disabled
+  - match: { test_modify_source_mode_synthetic_disabled.mappings._source.mode: synthetic }
+
+---
+modify logsdb index source mode to disabled after index creation:
+  - do:
+      indices.create:
+        index: test_modify_logsdb_disabled_after_creation
+        body:
+          settings:
+            index:
+              mode: logsdb
+
+  - do:
+      catch: bad_request
+      indices.put_mapping:
+        index: test_modify_logsdb_disabled_after_creation
+        body:
+          _source:
+            mode: disabled
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" }
+
+---
+modify logsdb index source mode to stored after index creation:
+  - do:
+      indices.create:
+        index: test_modify_logsdb_stored_after_creation
+        body:
+          settings:
+            index:
+              mode: logsdb
+
+  - do:
+      catch: bad_request
+      indices.put_mapping:
+        index: test_modify_logsdb_stored_after_creation
+        body:
+          _source:
+            mode: stored
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" }
+
+---
+modify time_series index source mode to disabled after index creation:
+  - do:
+      indices.create:
+        index: test_modify_time_series_disabled_after_creation
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      catch: bad_request
+      indices.put_mapping:
+        index: test_modify_time_series_disabled_after_creation
+        body:
+          _source:
+            mode: disabled
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" }
+
+---
+modify time_series index source mode to stored after index creation:
+  - do:
+      indices.create:
+        index: test_modify_time_series_stored_after_creation
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [ keyword ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      catch: bad_request
+      indices.put_mapping:
+        index: test_modify_time_series_stored_after_creation
+        body:
+          _source:
+            mode: stored
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" }
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml
new file mode 100644
index 0000000000000..bc81d1eb67309
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml
@@ -0,0 +1,381 @@
+---
+setup:
+  - requires:
+      test_runner_features: allowed_warnings_regex
+
+  - do:
+      indices.create:
+        index: my-index
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+              agent_id:
+                type: keyword
+                doc_values: false
+                store: false
+              process_id:
+                type: integer
+                doc_values: false
+                store: false
+              http_method:
+                type: keyword
+                doc_values: false
+                store: false
+              is_https:
+                type: boolean
+                doc_values: false
+                store: false
+              location:
+                type: geo_point
+                doc_values: false
+                store: false
+              message:
+                type: text
+                store: false
+                fields:
+                  raw:
+                    type: keyword
+
+  - do:
+      bulk:
+        index: my-index
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "That's no moon. It's a space station." }
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: my-index
+
+---
+"Simple from":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "agent_id"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "host.name"}
+  - match: {columns.2.type: "keyword"}
+  - match: {columns.3.name: "http_method" }
+  - match: {columns.3.type: "keyword" }
+  - match: {columns.4.name: "is_https"}
+  - match: {columns.4.type: "boolean"}
+  - match: {columns.5.name: "location"}
+  - match: {columns.5.type: "geo_point"}
+  - match: {columns.6.name: "message"}
+  - match: {columns.6.type: "text"}
+  - match: {columns.7.name: "message.raw"}
+  - match: {columns.7.type: "keyword"}
+  - match: {columns.8.name: "process_id"}
+  - match: {columns.8.type: "integer"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "yoda"}
+  - match: {values.0.2: "bar"}
+  - match: {values.0.3: "PUT"}
+  - match: {values.0.4: false}
+  - match: {values.0.5: "POINT (-74.006 40.7128)"}
+  - match: {values.0.6: "Do. Or do not. There is no try."}
+  - match: {values.0.7: "Do. Or do not. There is no try."}
+  - match: {values.0.8: 102}
+
+---
+"Simple from geo point":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP location | LIMIT 10'
+
+  - match: {columns.0.name: "location"}
+  - match: {columns.0.type: "geo_point"}
+
+  - match: {values.0.0: "POINT (-74.006 40.7128)"}
+  - match: {values.1.0: "POINT (-74.006 40.7128)"}
+  - match: {values.2.0: "POINT (-74.006 40.7128)"}
+  - match: {values.3.0: "POINT (-74.006 40.7128)"}
+  - match: {values.4.0: "POINT (-74.006 40.7128)"}
+  - match: {values.5.0: "POINT (-74.006 40.7128)"}
+
+---
+"Simple from number fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP process_id | LIMIT 10'
+
+  - match: {columns.0.name: "process_id"}
+  - match: {columns.0.type: "integer"}
+
+  - match: {values.0.0: 102}
+  - match: {values.1.0: 102}
+  - match: {values.2.0: 104}
+  - match: {values.3.0: 101}
+  - match: {values.4.0: 103}
+  - match: {values.5.0: 105}
+
+---
+"Simple from keyword fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP agent_id, http_method | LIMIT 10'
+
+  - match: {columns.0.name: "agent_id"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "http_method"}
+  - match: {columns.1.type: "keyword"}
+
+  - match: {values.0.0: "yoda"}
+  - match: {values.0.1: "PUT"}
+  - match: {values.1.0: "darth-vader"}
+  - match: {values.1.1: "POST"}
+  - match: {values.2.0: "yoda"}
+  - match: {values.2.1: "POST"}
+  - match: {values.3.0: "darth-vader"}
+  - match: {values.3.1: "GET"}
+  - match: {values.4.0: "obi-wan"}
+  - match: {values.4.1: "GET"}
+  - match: {values.5.0: "obi-wan"}
+  - match: {values.5.1: "GET"}
+
+---
+"Simple from boolean fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP is_https | LIMIT 10'
+
+  - match: {columns.0.name: "is_https"}
+  - match: {columns.0.type: "boolean"}
+
+  - match: {values.0.0: false}
+  - match: {values.1.0: true}
+  - match: {values.2.0: false}
+  - match: {values.3.0: false}
+  - match: {values.4.0: false}
+  - match: {values.5.0: false}
+
+---
+"Simple from text fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}
+
+---
+"message field without keyword multi-field":
+  - do:
+      indices.create:
+        index: my-index2
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+              agent_id:
+                type: keyword
+                doc_values: false
+                store: false
+              process_id:
+                type: integer
+                doc_values: false
+                store: false
+              http_method:
+                type: keyword
+                doc_values: false
+                store: false
+              is_https:
+                type: boolean
+                doc_values: false
+                store: false
+              location:
+                type: geo_point
+                doc_values: false
+                store: false
+              message:
+                type: text
+                store: false
+
+  - do:
+      bulk:
+        index: my-index2
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "That's no moon. It's a space station." }
+
+  - do:
+      allowed_warnings_regex:
+        - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "agent_id"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "host.name"}
+  - match: {columns.2.type: "keyword"}
+  - match: {columns.3.name: "http_method" }
+  - match: {columns.3.type: "keyword" }
+  - match: {columns.4.name: "is_https"}
+  - match: {columns.4.type: "boolean"}
+  - match: {columns.5.name: "location"}
+  - match: {columns.5.type: "geo_point"}
+  - match: {columns.6.name: "message"}
+  - match: {columns.6.type: "text"}
+  - match: {columns.7.name: "process_id"}
+  - match: {columns.7.type: "integer"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "yoda"}
+  - match: {values.0.2: "bar"}
+  - match: {values.0.3: "PUT"}
+  - match: {values.0.4: false}
+  - match: {values.0.5: "POINT (-74.006 40.7128)"}
+  - match: {values.0.6: "Do. Or do not. There is no try."}
+  - match: {values.0.7: 102}
+
+  - do:
+      allowed_warnings_regex:
+        - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}
+
+---
+"message field with keyword multi-field with ignore_above":
+  - do:
+      indices.create:
+        index: my-index2
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+                store: false
+              message:
+                type: text
+                store: false
+                fields:
+                  raw:
+                    type: keyword
+                    ignore_above: 3
+
+  - do:
+      bulk:
+        index: my-index2
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "message": "That's no moon. It's a space station." }
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "host.name"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "message"}
+  - match: {columns.2.type: "text"}
+  - match: {columns.3.name: "message.raw"}
+  - match: {columns.3.type: "keyword"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "bar"}
+  - match: {values.0.2: "Do. Or do not. There is no try."}
+  # Note that isn't related to synthetic source. For both stored and synthetic source null is returned:
+#  - match: {values.0.3: "Do. Or do not. There is no try."}
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml
new file mode 100644
index 0000000000000..6c840a0cf9d3a
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml
@@ -0,0 +1,254 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: my-index
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              is_https:
+                type: boolean
+              location:
+                type: geo_point
+              message:
+                type: text
+
+  - do:
+      bulk:
+        index: my-index
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "That's no moon. It's a space station." }
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: my-index
+
+---
+"Simple from":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "agent_id"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "host.name"}
+  - match: {columns.2.type: "keyword"}
+  - match: {columns.3.name: "http_method" }
+  - match: {columns.3.type: "keyword" }
+  - match: {columns.4.name: "is_https"}
+  - match: {columns.4.type: "boolean"}
+  - match: {columns.5.name: "location"}
+  - match: {columns.5.type: "geo_point"}
+  - match: {columns.6.name: "message"}
+  - match: {columns.6.type: "text"}
+  - match: {columns.7.name: "process_id"}
+  - match: {columns.7.type: "integer"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "yoda"}
+  - match: {values.0.2: "bar"}
+  - match: {values.0.3: "PUT"}
+  - match: {values.0.4: false}
+  - match: {values.0.5: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.0.6: "Do. Or do not. There is no try."}
+  - match: {values.0.7: 102}
+
+---
+"Simple from geo point":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP location | LIMIT 10'
+
+  - match: {columns.0.name: "location"}
+  - match: {columns.0.type: "geo_point"}
+
+  - match: {values.0.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.1.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.2.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.3.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.4.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.5.0: "POINT (-74.00600004941225 40.712799984030426)"}
+
+---
+"Simple from number fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP process_id | LIMIT 10'
+
+  - match: {columns.0.name: "process_id"}
+  - match: {columns.0.type: "integer"}
+
+  - match: {values.0.0: 102}
+  - match: {values.1.0: 102}
+  - match: {values.2.0: 104}
+  - match: {values.3.0: 101}
+  - match: {values.4.0: 103}
+  - match: {values.5.0: 105}
+
+---
+"Simple from keyword fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP agent_id, http_method | LIMIT 10'
+
+  - match: {columns.0.name: "agent_id"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "http_method"}
+  - match: {columns.1.type: "keyword"}
+
+  - match: {values.0.0: "yoda"}
+  - match: {values.0.1: "PUT"}
+  - match: {values.1.0: "darth-vader"}
+  - match: {values.1.1: "POST"}
+  - match: {values.2.0: "yoda"}
+  - match: {values.2.1: "POST"}
+  - match: {values.3.0: "darth-vader"}
+  - match: {values.3.1: "GET"}
+  - match: {values.4.0: "obi-wan"}
+  - match: {values.4.1: "GET"}
+  - match: {values.5.0: "obi-wan"}
+  - match: {values.5.1: "GET"}
+
+---
+"Simple from boolean fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP is_https | LIMIT 10'
+
+  - match: {columns.0.name: "is_https"}
+  - match: {columns.0.type: "boolean"}
+
+  - match: {values.0.0: false}
+  - match: {values.1.0: true}
+  - match: {values.2.0: false}
+  - match: {values.3.0: false}
+  - match: {values.4.0: false}
+  - match: {values.5.0: false}
+
+---
+"Simple from text fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}
+
+---
+"message field with stored keyword multi-field with ignore_above":
+  - do:
+      indices.create:
+        index: my-index2
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+                store: false
+              message:
+                type: text
+                store: false
+                fields:
+                  raw:
+                    type: keyword
+                    store: true
+
+  - do:
+      bulk:
+        index: my-index2
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "message": "That's no moon. It's a space station." }
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "host.name"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "message"}
+  - match: {columns.2.type: "text"}
+  - match: {columns.3.name: "message.raw"}
+  - match: {columns.3.type: "keyword"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "bar"}
+  - match: {values.0.2: "Do. Or do not. There is no try."}
+  - match: {values.0.3: "Do. Or do not. There is no try."}
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}
+
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java
index 081a170aac9f1..bb4464542a422 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java
@@ -87,7 +87,7 @@ protected void doExecute(Task task, GetPipelineRequest request, ActionListener<G
             )
             .setScroll(TimeValue.timeValueMinutes(1L))
             .execute(ActionListener.wrap(searchResponse -> {
-                final int numHits = Math.toIntExact(searchResponse.getHits().getTotalHits().value);
+                final int numHits = Math.toIntExact(searchResponse.getHits().getTotalHits().value());
                 final Map<String, BytesReference> pipelineSources = Maps.newMapWithExpectedSize(numHits);
                 final Consumer<SearchResponse> clearScroll = (response) -> {
                     if (response != null && response.getScrollId() != null) {
@@ -148,14 +148,14 @@ private void handleFilteringSearchResponse(
         ActionListener<GetPipelineResponse> listener
     ) {
         int numberOfHitsSeenSoFar = numberOfHitsSeenPreviously + searchResponse.getHits().getHits().length;
-        if (numberOfHitsSeenSoFar > searchResponse.getHits().getTotalHits().value) {
+        if (numberOfHitsSeenSoFar > searchResponse.getHits().getTotalHits().value()) {
             clearScroll.accept(searchResponse);
             listener.onFailure(
                 new IllegalStateException(
                     "scrolling returned more hits ["
                         + numberOfHitsSeenSoFar
                         + "] than expected ["
-                        + searchResponse.getHits().getTotalHits().value
+                        + searchResponse.getHits().getTotalHits().value()
                         + "] so bailing out to prevent unbounded "
                         + "memory consumption."
                 )
@@ -179,7 +179,7 @@ private void handleFilteringSearchResponse(
             }
         }
 
-        if (numberOfHitsSeenSoFar == searchResponse.getHits().getTotalHits().value) {
+        if (numberOfHitsSeenSoFar == searchResponse.getHits().getTotalHits().value()) {
             clearScroll.accept(searchResponse);
             listener.onResponse(new GetPipelineResponse(pipelineSources));
         } else {
diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java
index c594c9f553164..216f82552353b 100644
--- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java
+++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java
@@ -19,6 +19,7 @@
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.LevenshteinAutomata;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -291,7 +292,10 @@ public Query regexpQuery(
                 return new MatchNoDocsQuery();
             }
 
-            final Automaton automaton = new RegExp(regexp, syntaxFlags, matchFlags).toAutomaton(maxDeterminizedStates);
+            final Automaton automaton = Operations.determinize(
+                new RegExp(regexp, syntaxFlags, matchFlags).toAutomaton(),
+                maxDeterminizedStates
+            );
             final CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton);
             if (runAutomaton.run(this.value)) {
                 return new MatchAllDocsQuery();
diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java
index e52237f4d507e..3a50cc8143485 100644
--- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java
+++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java
@@ -242,11 +242,8 @@ public int docValueCount() {
 
         @Override
         public long nextOrd() {
-            if (ordsForThisDoc.hasNext()) {
-                return ordsForThisDoc.next();
-            } else {
-                return NO_MORE_ORDS;
-            }
+            assert ordsForThisDoc.hasNext();
+            return ordsForThisDoc.next();
         }
 
         @Override
diff --git a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldTypeTests.java b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldTypeTests.java
index c29e4513562fc..04599549cc3cc 100644
--- a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldTypeTests.java
+++ b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldTypeTests.java
@@ -77,11 +77,7 @@ private CollectionBasedSortedSetDocValues(List<BytesRef> docValues) {
 
         @Override
         public long nextOrd() {
-            currentOrd++;
-            if (currentOrd >= docValues.size()) {
-                return NO_MORE_ORDS;
-            }
-            return currentOrd;
+            return ++currentOrd;
         }
 
         @Override
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
index 5b04225cee105..b43d87c17e644 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
@@ -650,7 +650,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
 
         List<Field> fields = new ArrayList<>();
         if (indexed && hasDocValues) {
-            fields.add(new LongField(fieldType().name(), numericValue));
+            fields.add(new LongField(fieldType().name(), numericValue, Field.Store.NO));
         } else if (hasDocValues) {
             fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue));
         } else if (indexed) {
diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java
index 00532d95574c0..4f42103bc4541 100644
--- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java
+++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 
 import java.util.Locale;
@@ -213,9 +212,9 @@ static CompiledAutomaton prefixAutomaton(String versionPrefix, boolean caseInsen
 
         a = Operations.concatenate(a, Automata.makeAnyBinary());
         assert a.isDeterministic();
-        a = MinimizationOperations.minimize(a, 0);
+        a = Operations.determinize(a, 0);
 
-        return new CompiledAutomaton(a, null, true, 0, true);
+        return new CompiledAutomaton(a, false, true, true);
     }
 
     static class EncodedVersion {
diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java
index 387a49a29dc23..1e5ecf19bdf81 100644
--- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java
+++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java
@@ -40,11 +40,11 @@ class VersionFieldWildcardQuery extends AutomatonQuery {
     private static final byte WILDCARD_CHAR = '?';
 
     VersionFieldWildcardQuery(Term term, boolean caseInsensitive) {
-        super(term, toAutomaton(term, caseInsensitive), Integer.MAX_VALUE, true);
+        super(term, toAutomaton(term, caseInsensitive), true);
     }
 
     VersionFieldWildcardQuery(Term term, boolean caseInsensitive, RewriteMethod rewriteMethod) {
-        super(term, toAutomaton(term, caseInsensitive), Integer.MAX_VALUE, true, rewriteMethod);
+        super(term, toAutomaton(term, caseInsensitive), true, rewriteMethod);
     }
 
     private static Automaton toAutomaton(Term wildcardquery, boolean caseInsensitive) {
@@ -114,7 +114,7 @@ private static Automaton toAutomaton(Term wildcardquery, boolean caseInsensitive
         if (containsPreReleaseSeparator == false) {
             automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELEASE_SEPARATOR_BYTE)));
         }
-        return Operations.concatenate(automata);
+        return Operations.determinize(Operations.concatenate(automata), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     @Override
diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringDocValuesField.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringDocValuesField.java
index 17e1d70cbb471..01f0fdb256551 100644
--- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringDocValuesField.java
+++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringDocValuesField.java
@@ -47,7 +47,8 @@ public VersionStringDocValuesField(SortedSetDocValues input, String name) {
     public void setNextDocId(int docId) throws IOException {
         count = 0;
         if (input.advanceExact(docId)) {
-            for (long ord = input.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = input.nextOrd()) {
+            for (int i = 0; i < input.docValueCount(); i++) {
+                long ord = input.nextOrd();
                 ords = ArrayUtil.grow(ords, count + 1);
                 ords[count++] = ord;
             }
diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
index b49b4500ce7b7..6bf2917c601ac 100644
--- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
+++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
@@ -187,7 +187,8 @@ public Query regexpQuery(
                 matchFlags,
                 DEFAULT_PROVIDER,
                 maxDeterminizedStates,
-                method == null ? CONSTANT_SCORE_REWRITE : method
+                method == null ? CONSTANT_SCORE_REWRITE : method,
+                true
             ) {
 
                 @Override
diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java
index 94d8a144b0bd6..c89d1f8493b6b 100644
--- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java
+++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java
@@ -117,7 +117,7 @@ public void testPrefixQuery() throws IOException {
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1.0-A").caseInsensitive(true)),
             response -> {
-                assertEquals(1, response.getHits().getTotalHits().value);
+                assertEquals(1, response.getHits().getTotalHits().value());
                 assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[0].getSourceAsMap().get("version"));
             }
         );
@@ -134,7 +134,7 @@ public void testSort() throws IOException {
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.DESC),
             response -> {
-                assertEquals(8, response.getHits().getTotalHits().value);
+                assertEquals(8, response.getHits().getTotalHits().value());
                 SearchHit[] hits = response.getHits().getHits();
                 assertEquals("1.3.567#12", hits[0].getSortValues()[0]);
                 assertEquals("1.2.3alpha", hits[1].getSortValues()[0]);
@@ -150,7 +150,7 @@ public void testSort() throws IOException {
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC),
             response -> {
-                assertEquals(8, response.getHits().getTotalHits().value);
+                assertEquals(8, response.getHits().getTotalHits().value());
                 var hits = response.getHits().getHits();
                 assertEquals("1.0.0", hits[0].getSortValues()[0]);
                 assertEquals("1.3.0+build.1234567", hits[1].getSortValues()[0]);
@@ -179,7 +179,7 @@ public void testRegexQuery() throws Exception {
         client().admin().indices().prepareRefresh(indexName).get();
 
         assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "2.*0")), response -> {
-            assertEquals(2, response.getHits().getTotalHits().value);
+            assertEquals(2, response.getHits().getTotalHits().value());
             assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version"));
             assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version"));
         });
@@ -187,21 +187,21 @@ public void testRegexQuery() throws Exception {
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "<0-10>.<0-10>.*al.*")),
             response -> {
-                assertEquals(2, response.getHits().getTotalHits().value);
+                assertEquals(2, response.getHits().getTotalHits().value());
                 assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
                 assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version"));
             }
         );
 
         assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "1.[0-9].[0-9].*")), response -> {
-            assertEquals(2, response.getHits().getTotalHits().value);
+            assertEquals(2, response.getHits().getTotalHits().value());
             assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
             assertEquals("1.3.0+build.1234567", response.getHits().getHits()[1].getSourceAsMap().get("version"));
         });
 
         // test case sensitivity / insensitivity
         assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", ".*alpha.*")), response -> {
-            assertEquals(2, response.getHits().getTotalHits().value);
+            assertEquals(2, response.getHits().getTotalHits().value());
             assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
             assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version"));
         });
@@ -211,7 +211,7 @@ public void testRegexQuery() throws Exception {
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", ".*Alpha.*").caseInsensitive(true)),
             response -> {
-                assertEquals(2, response.getHits().getTotalHits().value);
+                assertEquals(2, response.getHits().getTotalHits().value());
                 assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
                 assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version"));
             }
@@ -234,7 +234,7 @@ public void testFuzzyQuery() throws Exception {
         client().admin().indices().prepareRefresh(indexName).get();
 
         assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.fuzzyQuery("version", "2.3.0")), response -> {
-            assertEquals(3, response.getHits().getTotalHits().value);
+            assertEquals(3, response.getHits().getTotalHits().value());
             assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version"));
             assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version"));
             assertEquals("2.a3.0", response.getHits().getHits()[2].getSourceAsMap().get("version"));
@@ -288,7 +288,7 @@ public void testWildcardQuery() throws Exception {
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*Alpha*").caseInsensitive(true)),
             response -> {
-                assertEquals(2, response.getHits().getTotalHits().value);
+                assertEquals(2, response.getHits().getTotalHits().value());
                 assertEquals("1.0.0-alpha.2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version"));
                 assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version"));
             }
@@ -297,7 +297,7 @@ public void testWildcardQuery() throws Exception {
 
     private void checkWildcardQuery(String indexName, String query, String... expectedResults) {
         assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", query)), response -> {
-            assertEquals(expectedResults.length, response.getHits().getTotalHits().value);
+            assertEquals(expectedResults.length, response.getHits().getTotalHits().value());
             for (int i = 0; i < expectedResults.length; i++) {
                 String expected = expectedResults[i];
                 Object actual = response.getHits().getHits()[i].getSourceAsMap().get("version");
@@ -321,7 +321,7 @@ public void testStoreMalformed() throws Exception {
         client().admin().indices().prepareRefresh(indexName).get();
 
         assertResponse(client().prepareSearch(indexName).addDocValueField("version"), response -> {
-            assertEquals(4, response.getHits().getTotalHits().value);
+            assertEquals(4, response.getHits().getTotalHits().value());
             assertEquals("1", response.getHits().getAt(0).getId());
             assertEquals("1.invalid.0", response.getHits().getAt(0).field("version").getValue());
 
@@ -359,7 +359,7 @@ public void testStoreMalformed() throws Exception {
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC),
             response -> {
-                assertEquals(4, response.getHits().getTotalHits().value);
+                assertEquals(4, response.getHits().getTotalHits().value());
                 SearchHit[] hits = response.getHits().getHits();
                 assertEquals("2.2.0", hits[0].getSortValues()[0]);
                 assertEquals("", hits[1].getSortValues()[0]);
@@ -437,36 +437,36 @@ public void testMultiValues() throws Exception {
         client().admin().indices().prepareRefresh(indexName).get();
 
         assertResponse(client().prepareSearch(indexName).addSort("version", SortOrder.ASC), response -> {
-            assertEquals(3, response.getHits().getTotalHits().value);
+            assertEquals(3, response.getHits().getTotalHits().value());
             assertEquals("1", response.getHits().getAt(0).getId());
             assertEquals("2", response.getHits().getAt(1).getId());
             assertEquals("3", response.getHits().getAt(2).getId());
         });
 
         assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "3.0.0")), response -> {
-            assertEquals(1, response.getHits().getTotalHits().value);
+            assertEquals(1, response.getHits().getTotalHits().value());
             assertEquals("1", response.getHits().getAt(0).getId());
         });
 
         assertResponse(client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "4.alpha.0")), response -> {
-            assertEquals(1, response.getHits().getTotalHits().value);
+            assertEquals(1, response.getHits().getTotalHits().value());
             assertEquals("2", response.getHits().getAt(0).getId());
         });
 
         // range
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.5.0")),
-            response -> assertEquals(1, response.getHits().getTotalHits().value)
+            response -> assertEquals(1, response.getHits().getTotalHits().value())
         );
 
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.5.0")),
-            response -> assertEquals(3, response.getHits().getTotalHits().value)
+            response -> assertEquals(3, response.getHits().getTotalHits().value())
         );
 
         assertResponse(
             client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("5.0.0").to("6.0.0")),
-            response -> assertEquals(1, response.getHits().getTotalHits().value)
+            response -> assertEquals(1, response.getHits().getTotalHits().value())
         );
     }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
index 51e39f144e44c..554cd0489ae8a 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
@@ -18,6 +18,7 @@ dependencies {
   javaRestTestImplementation project(path: xpackModule('esql-core'))
   javaRestTestImplementation project(path: xpackModule('esql'))
   javaRestTestImplementation project(path: xpackModule('snapshot-repo-test-kit'))
+  javaRestTestImplementation project(path: xpackModule('ent-search'))
 }
 
 // location for keys and certificates
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AdaptiveAllocationsScaleFromZeroIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AdaptiveAllocationsScaleFromZeroIT.java
new file mode 100644
index 0000000000000..db34ec5eedc86
--- /dev/null
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AdaptiveAllocationsScaleFromZeroIT.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.integration;
+
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseListener;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+@LuceneTestCase.AwaitsFix(bugUrl = "Cannot test without setting the scale to zero period to a small value")
+public class AdaptiveAllocationsScaleFromZeroIT extends PyTorchModelRestTestCase {
+
+    @SuppressWarnings("unchecked")
+    public void testScaleFromZero() throws Exception {
+        String modelId = "test_scale_from_zero";
+        createPassThroughModel(modelId);
+        putModelDefinition(modelId, PyTorchModelIT.BASE_64_ENCODED_MODEL, PyTorchModelIT.RAW_MODEL_SIZE);
+        putVocabulary(List.of("Auto", "scale", "and", "infer"), modelId);
+
+        startDeployment(modelId, modelId, new AdaptiveAllocationsSettings(true, 0, 1));
+        {
+            var responseMap = entityAsMap(getTrainedModelStats(modelId));
+            List<Map<String, Object>> stats = (List<Map<String, Object>>) responseMap.get("trained_model_stats");
+            String statusState = (String) XContentMapValues.extractValue("deployment_stats.allocation_status.state", stats.get(0));
+            assertThat(responseMap.toString(), statusState, is(not(nullValue())));
+            Integer count = (Integer) XContentMapValues.extractValue("deployment_stats.allocation_status.allocation_count", stats.get(0));
+            assertThat(responseMap.toString(), count, is(1));
+        }
+
+        // wait for scale down. The scaler service will check every 10 seconds
+        assertBusy(() -> {
+            var statsMap = entityAsMap(getTrainedModelStats(modelId));
+            List<Map<String, Object>> innerStats = (List<Map<String, Object>>) statsMap.get("trained_model_stats");
+            Integer innerCount = (Integer) XContentMapValues.extractValue(
+                "deployment_stats.allocation_status.allocation_count",
+                innerStats.get(0)
+            );
+            assertThat(statsMap.toString(), innerCount, is(0));
+        }, 30, TimeUnit.SECONDS);
+
+        var failures = new ConcurrentLinkedDeque<Exception>();
+
+        // infer will scale up
+        int inferenceCount = 10;
+        var latch = new CountDownLatch(inferenceCount);
+        for (int i = 0; i < inferenceCount; i++) {
+            asyncInfer("Auto scale and infer", modelId, TimeValue.timeValueSeconds(5), new ResponseListener() {
+                @Override
+                public void onSuccess(Response response) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception exception) {
+                    latch.countDown();
+                    failures.add(exception);
+                }
+            });
+        }
+
+        latch.await();
+        assertThat(failures, empty());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testMultipleDeploymentsWaiting() throws Exception {
+        String id1 = "test_scale_from_zero_dep_1";
+        String id2 = "test_scale_from_zero_dep_2";
+        String id3 = "test_scale_from_zero_dep_3";
+        var idsList = Arrays.asList(id1, id2, id3);
+        for (var modelId : idsList) {
+            createPassThroughModel(modelId);
+            putModelDefinition(modelId, PyTorchModelIT.BASE_64_ENCODED_MODEL, PyTorchModelIT.RAW_MODEL_SIZE);
+            putVocabulary(List.of("Auto", "scale", "and", "infer"), modelId);
+
+            startDeployment(modelId, modelId, new AdaptiveAllocationsSettings(true, 0, 1));
+        }
+
+        // wait for scale down. The scaler service will check every 10 seconds
+        assertBusy(() -> {
+            var statsMap = entityAsMap(getTrainedModelStats("test_scale_from_zero_dep_*"));
+            List<Map<String, Object>> innerStats = (List<Map<String, Object>>) statsMap.get("trained_model_stats");
+            assertThat(innerStats, hasSize(3));
+            for (int i = 0; i < 3; i++) {
+                Integer innerCount = (Integer) XContentMapValues.extractValue(
+                    "deployment_stats.allocation_status.allocation_count",
+                    innerStats.get(i)
+                );
+                assertThat(statsMap.toString(), innerCount, is(0));
+            }
+        }, 30, TimeUnit.SECONDS);
+
+        // infer will scale up
+        int inferenceCount = 10;
+        var latch = new CountDownLatch(inferenceCount);
+        for (int i = 0; i < inferenceCount; i++) {
+            asyncInfer("Auto scale and infer", randomFrom(idsList), TimeValue.timeValueSeconds(5), new ResponseListener() {
+                @Override
+                public void onSuccess(Response response) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception exception) {
+                    latch.countDown();
+                    fail(exception.getMessage());
+                }
+            });
+        }
+
+        latch.await();
+    }
+}
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java
index 13fb2f21bbf67..f6eb7206009fa 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.autoscaling.action.PutAutoscalingPolicyAction;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResults;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelDefinitionPartAction;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelVocabularyAction;
@@ -62,14 +63,14 @@ public class AutoscalingIT extends MlNativeAutodetectIntegTestCase {
     @Before
     public void putSettings() {
         updateClusterSettings(
-            Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 100).put("logger.org.elasticsearch.xpack.ml", "DEBUG")
+            Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), 100).put("logger.org.elasticsearch.xpack.ml", "DEBUG")
         );
     }
 
     @After
     public void removeSettings() {
         updateClusterSettings(
-            Settings.builder().putNull(MachineLearning.MAX_LAZY_ML_NODES.getKey()).putNull("logger.org.elasticsearch.xpack.ml")
+            Settings.builder().putNull(MachineLearningField.MAX_LAZY_ML_NODES.getKey()).putNull("logger.org.elasticsearch.xpack.ml")
         );
         cleanUp();
     }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
index 8c245a4543abe..39519dc7931d0 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
@@ -345,7 +345,7 @@ private void testExpiredDeletion(Float customThrottle, int numUnusedState) throw
         assertResponse(
             prepareSearch(AnomalyDetectorsIndex.jobStateIndexPattern()).setFetchSource(false).setTrackTotalHits(true).setSize(10000),
             stateDocsResponse -> {
-                assertThat(stateDocsResponse.getHits().getTotalHits().value, greaterThanOrEqualTo(5L));
+                assertThat(stateDocsResponse.getHits().getTotalHits().value(), greaterThanOrEqualTo(5L));
 
                 int nonExistingJobDocsCount = 0;
                 List<String> nonExistingJobExampleIds = new ArrayList<>();
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
index bb322cf79adc0..d981a60adbdb5 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
@@ -747,7 +747,15 @@ public void testMultiIndexDelete() throws Exception {
                 }
               }
             }""", AnomalyDetectorsIndex.jobResultsAliasedName(jobId), Job.ID, jobId));
-        client().performRequest(extraIndex1);
+
+        // Creating an index with a leading dot (".") is now deprecated.
+        // Ensure the ensuing warning exception doesn't cause a test case failure
+        try {
+            client().performRequest(extraIndex1);
+        } catch (org.elasticsearch.client.WarningFailureException e) {
+            logger.warn(e.getMessage());
+        }
+
         Request extraIndex2 = new Request("PUT", indexName + "-002");
         extraIndex2.setJsonEntity(Strings.format("""
             {
@@ -762,7 +770,14 @@ public void testMultiIndexDelete() throws Exception {
                 }
               }
             }""", AnomalyDetectorsIndex.jobResultsAliasedName(jobId), Job.ID, jobId));
-        client().performRequest(extraIndex2);
+
+        // Creating an index with a leading dot (".") is now deprecated.
+        // Ensure the ensuing warning exception doesn't cause a test case failure
+        try {
+            client().performRequest(extraIndex2);
+        } catch (org.elasticsearch.client.WarningFailureException e) {
+            logger.warn(e.getMessage());
+        }
 
         // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
         String indicesBeforeDelete = EntityUtils.toString(
@@ -983,10 +998,10 @@ private Response openJob(String jobId, Optional<TimeValue> timeout) throws IOExc
     }
 
     private void closeJob(String jobId) throws IOException {
-        Response openResponse = client().performRequest(
+        Response closeResponse = client().performRequest(
             new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_close")
         );
-        assertThat(entityAsMap(openResponse), hasEntry("closed", true));
+        assertThat(entityAsMap(closeResponse), hasEntry("closed", true));
     }
 
     private Response putJob(String jobId, String jsonBody) throws IOException {
@@ -1000,5 +1015,8 @@ public void clearMlState() throws Exception {
         new MlRestTestStateCleaner(logger, adminClient()).resetFeatures();
         // Don't check analytics jobs as they are independent of anomaly detection jobs and should not be created by this test.
         waitForPendingTasks(adminClient(), taskName -> taskName.contains(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME));
+        // Finally, clean up any lingering persistent tasks (such as "_close", "_close[n]" etc.) that may negatively
+        // impact subsequent tests.
+        client().performRequest(new Request("POST", "/_tasks/_cancel"));
     }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
index 2e096f3262cb6..9864c88d1405c 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
@@ -372,7 +372,7 @@ protected long countForecastDocs(String jobId, String forecastId) {
                         .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId))
                         .filter(QueryBuilders.termQuery(Forecast.FORECAST_ID.getPreferredName(), forecastId))
                 ),
-            searchResponse -> count.set(searchResponse.getHits().getTotalHits().value)
+            searchResponse -> count.set(searchResponse.getHits().getTotalHits().value())
         );
         return count.get();
     }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java
index 94bc3150cb12e..5f82d996c87fa 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java
@@ -77,7 +77,7 @@ public void testPersistJobOnGracefulShutdown_givenTimeAdvancedAfterNoNewData() t
                         ++numStateRecords;
                     }
                 }
-                assertThat(stateDocsResponse1.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(stateDocsResponse1.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(numQuantileRecords, equalTo(1));
                 assertThat(numStateRecords, equalTo(1));
             }
@@ -117,7 +117,7 @@ public void testPersistJobOnGracefulShutdown_givenTimeAdvancedAfterNoNewData() t
                     }
                 }
 
-                assertThat(stateDocsResponse2.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(stateDocsResponse2.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(numQuantileRecords, equalTo(1));
                 assertThat(numStateRecords, equalTo(2));
 
@@ -154,7 +154,7 @@ public void testPersistJobOnGracefulShutdown_givenNoDataAndTimeAdvanced() throws
                         ++numStateRecords;
                     }
                 }
-                assertThat(stateDocsResponse.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(stateDocsResponse.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(numQuantileRecords, equalTo(1));
                 assertThat(numStateRecords, equalTo(1));
             }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java
index c785ae96c5c16..b89f1d0c77ed8 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java
@@ -10,13 +10,17 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseListener;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus;
 import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
 import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
@@ -282,6 +286,27 @@ protected Response startDeployment(
         return client().performRequest(request);
     }
 
+    protected Response startDeployment(String modelId, String deploymentId, AdaptiveAllocationsSettings adaptiveAllocationsSettings)
+        throws IOException {
+        String endPoint = "/_ml/trained_models/"
+            + modelId
+            + "/deployment/_start"
+            + "?deployment_id="
+            + deploymentId
+            + "&threads_per_allocation=1"
+            + "&wait_for=started";
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        builder.field("adaptive_allocations", adaptiveAllocationsSettings);
+        builder.endObject();
+        var body = Strings.toString(builder);
+
+        Request request = new Request("POST", endPoint);
+        request.setJsonEntity(body);
+        return client().performRequest(request);
+    }
+
     protected void stopDeployment(String modelId) throws IOException {
         stopDeployment(modelId, false, false);
     }
@@ -325,6 +350,14 @@ protected Response infer(String input, String modelId, TimeValue timeout) throws
         return client().performRequest(request);
     }
 
+    protected void asyncInfer(String input, String modelId, TimeValue timeout, ResponseListener responseListener) throws IOException {
+        Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=" + timeout.toString());
+        request.setJsonEntity(Strings.format("""
+            {  "docs": [{"input":"%s"}] }
+            """, input));
+        client().performRequestAsync(request, responseListener);
+    }
+
     protected Response infer(String input, String modelId) throws IOException {
         Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=30s");
         request.setJsonEntity(Strings.format("""
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
index f5d0b23b437f3..8a6499ec3bb6a 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
@@ -164,7 +164,7 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
                     + testDocsWithEmptyFeatureImportance
                     + "] test docs with empty feature importance"
                     + " from "
-                    + sourceData.getHits().getTotalHits().value
+                    + sourceData.getHits().getTotalHits().value()
                     + " hits.\n"
                     + badDocuments,
                 trainingDocsWithEmptyFeatureImportance + testDocsWithEmptyFeatureImportance,
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java
index 260a5dea0a3c1..388583f6f8656 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java
@@ -295,7 +295,7 @@ private Quantiles getQuantiles(String jobId) throws Exception {
             prepareSearch(".ml-state*").setQuery(QueryBuilders.idsQuery().addIds(Quantiles.documentId(jobId))).setSize(1),
             response -> {
                 SearchHits hits = response.getHits();
-                assertThat(hits.getTotalHits().value, equalTo(1L));
+                assertThat(hits.getTotalHits().value(), equalTo(1L));
                 try (
                     XContentParser parser = JsonXContent.jsonXContent.createParser(
                         XContentParserConfiguration.EMPTY,
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
index 8fbad7ccd3877..1505d374dfa08 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
@@ -396,7 +396,7 @@ public void testStopOutlierDetectionWithEnoughDocumentsToScroll() throws Excepti
         }
 
         assertResponse(prepareSearch(config.getDest().getIndex()).setTrackTotalHits(true), searchResponse -> {
-            if (searchResponse.getHits().getTotalHits().value == docCount) {
+            if (searchResponse.getHits().getTotalHits().value() == docCount) {
                 long seenCount = SearchResponseUtils.getTotalHitsValue(
                     prepareSearch(config.getDest().getIndex()).setTrackTotalHits(true)
                         .setQuery(QueryBuilders.existsQuery("custom_ml.outlier_score"))
@@ -404,7 +404,7 @@ public void testStopOutlierDetectionWithEnoughDocumentsToScroll() throws Excepti
                 logger.debug("We stopped during analysis: [{}] < [{}]", seenCount, docCount);
                 assertThat(seenCount, lessThan((long) docCount));
             } else {
-                logger.debug("We stopped during reindexing: [{}] < [{}]", searchResponse.getHits().getTotalHits().value, docCount);
+                logger.debug("We stopped during reindexing: [{}] < [{}]", searchResponse.getHits().getTotalHits().value(), docCount);
             }
         });
 
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java
index c15750de3b336..edc851def4468 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BucketCorrelationAggregationIT.java
@@ -77,7 +77,7 @@ public void testCountCorrelation() {
                 .setSize(0)
                 .setTrackTotalHits(true),
             percentilesSearch -> {
-                long totalHits = percentilesSearch.getHits().getTotalHits().value;
+                long totalHits = percentilesSearch.getHits().getTotalHits().value();
                 Percentiles percentiles = percentilesSearch.getAggregations().get("percentiles");
                 Tuple<RangeAggregationBuilder, BucketCorrelationAggregationBuilder> aggs = buildRangeAggAndSetExpectations(
                     percentiles,
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java
index 8fddfa47c377c..139d1b074c7b2 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java
@@ -192,7 +192,7 @@ private boolean doesLocalAuditMessageExist(String message) {
                 .setQuery(new MatchPhraseQueryBuilder("message", message))
                 .get();
             try {
-                return response.getHits().getTotalHits().value > 0;
+                return response.getHits().getTotalHits().value() > 0;
             } finally {
                 response.decRef();
             }
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
index 17fe20c5115ff..dfb960794537b 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
@@ -767,7 +767,7 @@ private static DataCounts getDataCountsFromIndex(String jobId) throws IOExceptio
             prepareSearch().setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
                 .setQuery(QueryBuilders.idsQuery().addIds(DataCounts.documentId(jobId))),
             searchResponse -> {
-                if (searchResponse.getHits().getTotalHits().value != 1) {
+                if (searchResponse.getHits().getTotalHits().value() != 1) {
                     setOnce.set(new DataCounts(jobId));
                     return;
                 }
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
index f6a58002bbac5..083a444fbf14c 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
@@ -84,9 +85,9 @@ public void testLazyNodeValidation() throws Exception {
         logger.info("Started [{}] nodes", numNodes);
         ensureStableCluster(numNodes);
         ensureTemplatesArePresent();
-        logger.info("[{}] is [{}]", MachineLearning.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes);
+        logger.info("[{}] is [{}]", MachineLearningField.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes);
         // Set our lazy node number
-        updateClusterSettings(Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes));
+        updateClusterSettings(Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes));
         // create and open first job, which succeeds:
         Job.Builder job = createJob("lazy-node-validation-job-1", ByteSizeValue.ofMb(2));
         PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index f8a590a23a2c1..6d21654f9e161 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -649,14 +649,6 @@ public void loadExtensions(ExtensionLoader loader) {
         Property.NodeScope
     );
 
-    public static final Setting<Integer> MAX_LAZY_ML_NODES = Setting.intSetting(
-        "xpack.ml.max_lazy_ml_nodes",
-        0,
-        0,
-        Property.OperatorDynamic,
-        Property.NodeScope
-    );
-
     // Before 8.0.0 this needs to match the max allowed value for xpack.ml.max_open_jobs,
     // as the current node could be running in a cluster where some nodes are still using
     // that setting. From 8.0.0 onwards we have the flexibility to increase it...
@@ -810,7 +802,7 @@ public List<Setting<?>> getSettings() {
             PROCESS_CONNECT_TIMEOUT,
             CONCURRENT_JOB_ALLOCATIONS,
             MachineLearningField.MAX_MODEL_MEMORY_LIMIT,
-            MAX_LAZY_ML_NODES,
+            MachineLearningField.MAX_LAZY_ML_NODES,
             MAX_MACHINE_MEMORY_PERCENT,
             AutodetectBuilder.MAX_ANOMALY_RECORDS_SETTING_DYNAMIC,
             MAX_OPEN_JOBS_PER_NODE,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java
index 5603e9c4dca8d..a81a08519f157 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java
@@ -11,9 +11,11 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService;
+import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService;
 import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 
@@ -27,7 +29,9 @@ public TransportExternalInferModelAction(
         ClusterService clusterService,
         XPackLicenseState licenseState,
         TrainedModelProvider trainedModelProvider,
-        AdaptiveAllocationsScalerService adaptiveAllocationsScalerService
+        AdaptiveAllocationsScalerService adaptiveAllocationsScalerService,
+        TrainedModelAssignmentService assignmentService,
+        ThreadPool threadPool
     ) {
         super(
             InferModelAction.EXTERNAL_NAME,
@@ -38,7 +42,9 @@ public TransportExternalInferModelAction(
             clusterService,
             licenseState,
             trainedModelProvider,
-            adaptiveAllocationsScalerService
+            adaptiveAllocationsScalerService,
+            assignmentService,
+            threadPool
         );
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
index 980cdc09252cb..9ebc510af4f4d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
@@ -220,7 +220,7 @@ static GetDeploymentStatsAction.Response addFailedRoutes(
 
                 // add nodes from the failures that were not in the task responses
                 for (var nodeRoutingState : nodeToRoutingStates.entrySet()) {
-                    if (visitedNodes.contains(nodeRoutingState.getKey()) == false) {
+                    if ((visitedNodes.contains(nodeRoutingState.getKey()) == false) && nodes.nodeExists(nodeRoutingState.getKey())) {
                         updatedNodeStats.add(
                             AssignmentStats.NodeStats.forNotStartedState(
                                 nodes.get(nodeRoutingState.getKey()),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java
index a47b67e490851..210973f2601d3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java
@@ -189,7 +189,7 @@ private void initChunkedBucketSearcher(
             ML_ORIGIN,
             searchRequest,
             ActionListener.<SearchResponse>wrap(searchResponse -> {
-                long totalHits = searchResponse.getHits().getTotalHits().value;
+                long totalHits = searchResponse.getHits().getTotalHits().value();
                 if (totalHits > 0) {
                     InternalAggregations aggregations = searchResponse.getAggregations();
                     Min min = aggregations.get(EARLIEST_TIME);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
index b69f8c7d62eb2..ba4483493da1d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
@@ -42,7 +43,10 @@
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.inference.InferenceWaitForAllocation;
 import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService;
+import org.elasticsearch.xpack.ml.inference.adaptiveallocations.ScaleFromZeroFeatureFlag;
+import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService;
 import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel;
 import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
@@ -68,6 +72,8 @@ public class TransportInternalInferModelAction extends HandledTransportAction<Re
     private final XPackLicenseState licenseState;
     private final TrainedModelProvider trainedModelProvider;
     private final AdaptiveAllocationsScalerService adaptiveAllocationsScalerService;
+    private final InferenceWaitForAllocation waitForAllocation;
+    private final ThreadPool threadPool;
 
     TransportInternalInferModelAction(
         String actionName,
@@ -78,7 +84,9 @@ public class TransportInternalInferModelAction extends HandledTransportAction<Re
         ClusterService clusterService,
         XPackLicenseState licenseState,
         TrainedModelProvider trainedModelProvider,
-        AdaptiveAllocationsScalerService adaptiveAllocationsScalerService
+        AdaptiveAllocationsScalerService adaptiveAllocationsScalerService,
+        TrainedModelAssignmentService assignmentService,
+        ThreadPool threadPool
     ) {
         super(actionName, transportService, actionFilters, InferModelAction.Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.modelLoadingService = modelLoadingService;
@@ -87,6 +95,8 @@ public class TransportInternalInferModelAction extends HandledTransportAction<Re
         this.licenseState = licenseState;
         this.trainedModelProvider = trainedModelProvider;
         this.adaptiveAllocationsScalerService = adaptiveAllocationsScalerService;
+        this.waitForAllocation = new InferenceWaitForAllocation(assignmentService, this::inferOnBlockedRequest);
+        this.threadPool = threadPool;
     }
 
     @Inject
@@ -98,7 +108,9 @@ public TransportInternalInferModelAction(
         ClusterService clusterService,
         XPackLicenseState licenseState,
         TrainedModelProvider trainedModelProvider,
-        AdaptiveAllocationsScalerService adaptiveAllocationsScalerService
+        AdaptiveAllocationsScalerService adaptiveAllocationsScalerService,
+        TrainedModelAssignmentService assignmentService,
+        ThreadPool threadPool
     ) {
         this(
             InferModelAction.NAME,
@@ -109,7 +121,9 @@ public TransportInternalInferModelAction(
             clusterService,
             licenseState,
             trainedModelProvider,
-            adaptiveAllocationsScalerService
+            adaptiveAllocationsScalerService,
+            assignmentService,
+            threadPool
         );
     }
 
@@ -251,11 +265,11 @@ private void inferAgainstAllocatedModel(
 
         // Get a list of nodes to send the requests to and the number of
         // documents for each node.
-        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(request.numberOfDocuments(), RoutingState.STARTED);
+        var nodes = assignment.selectRandomNodesWeighedOnAllocations(request.numberOfDocuments(), RoutingState.STARTED);
 
         // We couldn't find any nodes in the started state so let's look for ones that are stopping in case we're shutting down some nodes
         if (nodes.isEmpty()) {
-            nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(request.numberOfDocuments(), RoutingState.STOPPING);
+            nodes = assignment.selectRandomNodesWeighedOnAllocations(request.numberOfDocuments(), RoutingState.STOPPING);
         }
 
         if (nodes.isEmpty()) {
@@ -263,7 +277,15 @@ private void inferAgainstAllocatedModel(
             boolean starting = adaptiveAllocationsScalerService.maybeStartAllocation(assignment);
             if (starting) {
                 message += "; starting deployment of one allocation";
+
+                if (ScaleFromZeroFeatureFlag.isEnabled()) {
+                    waitForAllocation.waitForAssignment(
+                        new InferenceWaitForAllocation.WaitingRequest(request, responseBuilder, parentTaskId, listener)
+                    );
+                    return;
+                }
             }
+
             logger.debug(message);
             listener.onFailure(ExceptionsHelper.conflictStatusException(message));
             return;
@@ -271,7 +293,42 @@ private void inferAgainstAllocatedModel(
 
         assert nodes.stream().mapToInt(Tuple::v2).sum() == request.numberOfDocuments()
             : "mismatch; sum of node requests does not match number of documents in request";
+        inferOnAssignmentNodes(assignment.getDeploymentId(), nodes, request, responseBuilder, parentTaskId, listener);
+    }
+
+    private void inferOnBlockedRequest(InferenceWaitForAllocation.WaitingRequest request, TrainedModelAssignment assignment) {
+        threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> {
 
+            var nodes = assignment.selectRandomNodesWeighedOnAllocations(request.request().numberOfDocuments(), RoutingState.STARTED);
+
+            if (nodes.isEmpty()) {
+                request.listener()
+                    .onFailure(
+                        new IllegalStateException(
+                            "[" + request.deploymentId() + "] error waiting for started allocations. The assignment has 0 started nodes"
+                        )
+                    );
+            }
+
+            inferOnAssignmentNodes(
+                assignment.getDeploymentId(),
+                nodes,
+                request.request(),
+                request.responseBuilder(),
+                request.parentTaskId(),
+                request.listener()
+            );
+        });
+    }
+
+    private void inferOnAssignmentNodes(
+        String deploymentId,
+        List<Tuple<String, Integer>> nodes,
+        Request request,
+        Response.Builder responseBuilder,
+        TaskId parentTaskId,
+        ActionListener<Response> listener
+    ) {
         AtomicInteger count = new AtomicInteger();
         AtomicArray<List<InferenceResults>> results = new AtomicArray<>(nodes.size());
         AtomicReference<Exception> failure = new AtomicReference<>();
@@ -282,14 +339,14 @@ private void inferAgainstAllocatedModel(
             InferTrainedModelDeploymentAction.Request deploymentRequest;
             if (request.getTextInput() == null) {
                 deploymentRequest = InferTrainedModelDeploymentAction.Request.forDocs(
-                    assignment.getDeploymentId(),
+                    deploymentId,
                     request.getUpdate(),
                     request.getObjectsToInfer().subList(startPos, startPos + node.v2()),
                     request.getInferenceTimeout()
                 );
             } else {
                 deploymentRequest = InferTrainedModelDeploymentAction.Request.forTextInput(
-                    assignment.getDeploymentId(),
+                    deploymentId,
                     request.getUpdate(),
                     request.getTextInput().subList(startPos, startPos + node.v2()),
                     request.getInferenceTimeout()
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
index bc017915e00aa..1edc02ff44a11 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.MlInfoAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -162,7 +163,7 @@ private Map<String, Object> limits() {
                 clusterSettings.get(MachineLearning.ALLOCATED_PROCESSORS_SCALE)
             );
             if (totalMlProcessors.count() > 0) {
-                int potentialExtraProcessors = Math.max(0, clusterSettings.get(MachineLearning.MAX_LAZY_ML_NODES) - mlNodes.size())
+                int potentialExtraProcessors = Math.max(0, clusterSettings.get(MachineLearningField.MAX_LAZY_ML_NODES) - mlNodes.size())
                     * singleNodeProcessors.roundUp();
                 limits.put("total_ml_processors", totalMlProcessors.roundUp() + potentialExtraProcessors);
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java
index 6aaa1e50f2e8a..d676e6cc9d065 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java
@@ -415,7 +415,7 @@ private void checkModelIdAgainstTags(String modelId, ActionListener<Void> listen
             ML_ORIGIN,
             searchRequest,
             ActionListener.<SearchResponse>wrap(response -> {
-                if (response.getHits().getTotalHits().value > 0) {
+                if (response.getHits().getTotalHits().value() > 0) {
                     listener.onFailure(
                         ExceptionsHelper.badRequestException(Messages.getMessage(Messages.INFERENCE_MODEL_ID_AND_TAGS_UNIQUE, modelId))
                     );
@@ -443,7 +443,7 @@ private void checkTagsAgainstModelIds(List<String> tags, ActionListener<Void> li
             ML_ORIGIN,
             searchRequest,
             ActionListener.<SearchResponse>wrap(response -> {
-                if (response.getHits().getTotalHits().value > 0) {
+                if (response.getHits().getTotalHits().value() > 0) {
                     listener.onFailure(
                         ExceptionsHelper.badRequestException(Messages.getMessage(Messages.INFERENCE_TAGS_AND_MODEL_IDS_UNIQUE, tags))
                     );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
index 2ec460a08caf9..759538b4cdc63 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
@@ -433,7 +433,7 @@ private static void checkDestIndexIsEmptyIfExists(
             TransportSearchAction.TYPE,
             destEmptySearch,
             ActionListener.wrap(searchResponse -> {
-                if (searchResponse.getHits().getTotalHits().value > 0) {
+                if (searchResponse.getHits().getTotalHits().value() > 0) {
                     listener.onFailure(ExceptionsHelper.badRequestException("dest index [{}] must be empty", destIndex));
                 } else {
                     listener.onResponse(startContext);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
index e130b13f4ec30..5fd70ce71cd24 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
@@ -234,9 +234,9 @@ protected void masterOperation(
             if (getModelResponse.getResources().results().size() > 1) {
                 listener.onFailure(
                     ExceptionsHelper.badRequestException(
-                        "cannot deploy more than one models at the same time; [{}] matches [{}] models]",
+                        "cannot deploy more than one model at the same time; [{}] matches models [{}]",
                         request.getModelId(),
-                        getModelResponse.getResources().results().size()
+                        getModelResponse.getResources().results().stream().map(TrainedModelConfig::getModelId).toList()
                     )
                 );
                 return;
@@ -671,7 +671,11 @@ public boolean test(ClusterState clusterState) {
                 deploymentId
             ).orElse(null);
             if (trainedModelAssignment == null) {
-                // Something weird happened, it should NEVER be null...
+                // The assignment may be null if it was stopped by another action while waiting
+                this.exception = new ElasticsearchStatusException(
+                    "Error waiting for the model deployment to start. The trained model assignment was removed while waiting",
+                    RestStatus.BAD_REQUEST
+                );
                 logger.trace(() -> format("[%s] assignment was null while waiting for state [%s]", deploymentId, waitForState));
                 return true;
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java
index cca59f27d5c76..dfe52897caf2c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingContext.java
@@ -177,7 +177,7 @@ public boolean isEmpty() {
         return anomalyDetectionTasks.isEmpty()
             && snapshotUpgradeTasks.isEmpty()
             && dataframeAnalyticsTasks.isEmpty()
-            && modelAssignments.isEmpty();
+            && modelAssignments.values().stream().allMatch(assignment -> assignment.getTaskParams().getNumberOfAllocations() == 0);
     }
 
     public List<String> findPartiallyAllocatedModels() {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java
index 18d974473251b..fee3d729f8dfd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java
@@ -30,6 +30,10 @@
 
 import static org.elasticsearch.core.Strings.format;
 
+/**
+ * This handles ML autoscaling just for classic cloud.
+ * For serverless, see: {@link MlAutoscalingResourceTracker}.
+ */
 public final class MlAutoscalingDeciderService implements AutoscalingDeciderService, LocalNodeMasterListener {
 
     private static final Logger logger = LogManager.getLogger(MlAutoscalingDeciderService.class);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java
index 9a9fbfa0340a9..6f14e2649a394 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java
@@ -48,7 +48,8 @@
 import static org.elasticsearch.xpack.ml.job.JobNodeSelector.AWAITING_LAZY_ASSIGNMENT;
 
 /**
- * backend for new kubernetes based autoscaler.
+ * This handles ML autoscaling just for serverless.
+ * For classic cloud, see: {@link MlAutoscalingDeciderService}.
  */
 public final class MlAutoscalingResourceTracker {
     private static final Logger logger = LogManager.getLogger(MlAutoscalingResourceTracker.class);
@@ -242,72 +243,72 @@ static void getMemoryAndProcessors(
             final int numMissingProcessors = numMissingAllocations * numberOfThreadsPerAllocation;
             int numExistingProcessorsToBeUsed = Math.min(numMissingProcessors, numberOfAvailableProcessors);
 
+            if (numberOfRequestedAllocations == 0) {
+                continue;
+            }
             if (assignment.getNodeRoutingTable().isEmpty() == false
                 && assignment.getNodeRoutingTable().values().stream().allMatch(r -> r.getState().consumesMemory() == false)) {
                 // Ignore states that don't consume memory, for example all allocations are failed or stopped
                 // if the node routing table is empty, then it will match the above condition, but it needs to be handled in the next branch
                 continue;
+            }
+
+            if (assignment.getNodeRoutingTable().isEmpty() == false) {
+                // if the routing table is non-empty, this is an existing model
+                existingModelMemoryBytes += estimatedMemoryUsage;
             } else {
+                // only increase memory requirements for new models
+                extraPerNodeModelMemoryBytes += Math.max(extraPerNodeModelMemoryBytes, estimatedMemoryUsage);
+                extraModelMemoryInBytes += estimatedMemoryUsage;
+            }
 
-                if (assignment.getNodeRoutingTable().isEmpty() == false) {
-                    // if the routing table is non-empty, this is an existing model
-                    existingModelMemoryBytes += estimatedMemoryUsage;
-                } else {
-                    // only increase memory requirements for new models
-                    extraPerNodeModelMemoryBytes += Math.max(extraPerNodeModelMemoryBytes, estimatedMemoryUsage);
-                    extraModelMemoryInBytes += estimatedMemoryUsage;
+            // if not low priority, check processor requirements.
+            if (Priority.LOW.equals(modelAssignment.getValue().getTaskParams().getPriority()) == false) {
+                if (numMissingProcessors > numberOfAvailableProcessors) {
+                    // as assignments can be placed on different nodes, we only need numberOfThreadsPerAllocation here
+                    extraProcessors += numMissingProcessors - numExistingProcessorsToBeUsed;
+                    extraPerNodeProcessors = Math.max(extraPerNodeProcessors, 1); // if extra processors >0, we need at least 1
+                                                                                  // extraPerNodeProcessors
                 }
-
-                // if not low priority, check processor requirements.
-                if (Priority.LOW.equals(modelAssignment.getValue().getTaskParams().getPriority()) == false) {
-                    if (numMissingProcessors > numberOfAvailableProcessors) {
-                        // as assignments can be placed on different nodes, we only need numberOfThreadsPerAllocation here
-                        extraProcessors += numMissingProcessors - numExistingProcessorsToBeUsed;
-                        extraPerNodeProcessors = Math.max(extraPerNodeProcessors, 1); // if extra processors >0, we need at least 1
-                                                                                      // extraPerNodeProcessors
-                    }
-                    if (perNodeAvailableProcessors < numberOfThreadsPerAllocation) {
-                        extraPerNodeProcessors = Math.max(extraPerNodeProcessors, numberOfThreadsPerAllocation);
-                    }
-                    numberOfAvailableProcessors -= numExistingProcessorsToBeUsed;
+                if (perNodeAvailableProcessors < numberOfThreadsPerAllocation) {
+                    extraPerNodeProcessors = Math.max(extraPerNodeProcessors, numberOfThreadsPerAllocation);
                 }
+                numberOfAvailableProcessors -= numExistingProcessorsToBeUsed;
+            }
+
+            if (extraProcessors > 0 || extraPerNodeProcessors > 0 || extraModelMemoryInBytes > 0 || extraPerNodeModelMemoryBytes > 0) {
+                logger.info(
+                    () -> format(
+                        "trained model [%s] assigned to [%s], waiting for [%d] allocations to start due to missing hardware",
+                        modelAssignment.getKey(),
+                        Strings.arrayToCommaDelimitedString(modelAssignment.getValue().getStartedNodes()),
+                        numMissingAllocations
+                    )
+                );
+            }
 
-                if (extraProcessors > 0 || extraPerNodeProcessors > 0 || extraModelMemoryInBytes > 0 || extraPerNodeModelMemoryBytes > 0) {
-                    logger.info(
-                        () -> format(
-                            "trained model [%s] assigned to [%s], waiting for [%d] allocations to start due to missing hardware",
-                            modelAssignment.getKey(),
-                            Strings.arrayToCommaDelimitedString(modelAssignment.getValue().getStartedNodes()),
-                            numMissingAllocations
+            for (String node : modelAssignment.getValue().getNodeRoutingTable().keySet()) {
+                sumOfCurrentlyExistingAndUsedProcessors += modelAssignment.getValue().getNodeRoutingTable().get(node).getTargetAllocations()
+                    * numberOfThreadsPerAllocation;
+
+                jobRequirementsByNode.computeIfAbsent(node, k -> new ArrayList<>())
+                    .add(
+                        MlJobRequirements.of(
+                            estimatedMemoryUsage,
+                            Priority.LOW.equals(modelAssignment.getValue().getTaskParams().getPriority())
+                                ? 0
+                                : modelAssignment.getValue().getNodeRoutingTable().get(node).getTargetAllocations()
+                                    * numberOfThreadsPerAllocation
                         )
                     );
-                }
-
-                for (String node : modelAssignment.getValue().getNodeRoutingTable().keySet()) {
-                    sumOfCurrentlyExistingAndUsedProcessors += modelAssignment.getValue()
-                        .getNodeRoutingTable()
-                        .get(node)
-                        .getTargetAllocations() * numberOfThreadsPerAllocation;
-
-                    jobRequirementsByNode.computeIfAbsent(node, k -> new ArrayList<>())
-                        .add(
-                            MlJobRequirements.of(
-                                estimatedMemoryUsage,
-                                Priority.LOW.equals(modelAssignment.getValue().getTaskParams().getPriority())
-                                    ? 0
-                                    : modelAssignment.getValue().getNodeRoutingTable().get(node).getTargetAllocations()
-                                        * numberOfThreadsPerAllocation
-                            )
-                        );
-                }
-
-                // min(3, max(number of allocations over all deployed models)
-                // the minimum number of nodes is equal to the number of allocations, up to 3
-                // if the number of allocations is greater than 3, then wantedMinNodes is still 3
-                // in theory this should help availability for 2-3 allocations
-                // the planner should split over all available nodes
-                minNodes = Math.min(3, Math.max(minNodes, numberOfRequestedAllocations));
             }
+
+            // min(3, max(number of allocations over all deployed models)
+            // the minimum number of nodes is equal to the number of allocations, up to 3
+            // if the number of allocations is greater than 3, then wantedMinNodes is still 3
+            // in theory this should help availability for 2-3 allocations
+            // the planner should split over all available nodes
+            minNodes = Math.min(3, Math.max(minNodes, numberOfRequestedAllocations));
         }
 
         // dummy autoscaling entity
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java
index f0e03a1e94973..7c41dbd463413 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java
@@ -62,7 +62,7 @@ public static DataExtractor.DataSummary getDataSummary(SearchResponse searchResp
         } else {
             Long earliestTime = toLongIfFinite((aggregations.<Min>get(EARLIEST_TIME)).value());
             Long latestTime = toLongIfFinite((aggregations.<Max>get(LATEST_TIME)).value());
-            long totalHits = searchResponse.getHits().getTotalHits().value;
+            long totalHits = searchResponse.getHits().getTotalHits().value();
             return new DataExtractor.DataSummary(earliestTime, latestTime, totalHits);
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java
index 20da61a3d6910..7829adb395675 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java
@@ -226,7 +226,7 @@ public void findDatafeedIdsForJobIds(Collection<String> jobIds, ActionListener<S
             listener.<SearchResponse>delegateFailureAndWrap((delegate, response) -> {
                 Set<String> datafeedIds = new HashSet<>();
                 // There cannot be more than one datafeed per job
-                assert response.getHits().getTotalHits().value <= jobIds.size();
+                assert response.getHits().getTotalHits().value() <= jobIds.size();
                 SearchHit[] hits = response.getHits().getHits();
 
                 for (SearchHit hit : hits) {
@@ -259,7 +259,7 @@ public void findDatafeedsByJobIds(
             listener.<SearchResponse>delegateFailureAndWrap((delegate, response) -> {
                 Map<String, DatafeedConfig.Builder> datafeedsByJobId = new HashMap<>();
                 // There cannot be more than one datafeed per job
-                assert response.getHits().getTotalHits().value <= jobIds.size();
+                assert response.getHits().getTotalHits().value() <= jobIds.size();
                 SearchHit[] hits = response.getHits().getHits();
                 for (SearchHit hit : hits) {
                     DatafeedConfig.Builder builder = parseLenientlyFromSource(hit.getSourceRef());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
index c890ab599c380..315d2249d00cb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
@@ -377,7 +377,7 @@ public DataSummary collectDataSummary() {
         SearchRequestBuilder searchRequestBuilder = buildDataSummarySearchRequestBuilder();
         SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder);
         try {
-            long rows = searchResponse.getHits().getTotalHits().value;
+            long rows = searchResponse.getHits().getTotalHits().value();
             LOGGER.debug(() -> format("[%s] Data summary rows [%s]", context.jobId, rows));
             return new DataSummary(rows, organicFeatures.length + processedFeatures.length);
         } finally {
@@ -396,7 +396,7 @@ public void collectDataSummaryAsync(ActionListener<DataSummary> dataSummaryActio
             TransportSearchAction.TYPE,
             searchRequestBuilder.request(),
             dataSummaryActionListener.delegateFailureAndWrap(
-                (l, searchResponse) -> l.onResponse(new DataSummary(searchResponse.getHits().getTotalHits().value, numberOfFields))
+                (l, searchResponse) -> l.onResponse(new DataSummary(searchResponse.getHits().getTotalHits().value(), numberOfFields))
             )
         );
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java
index dfcc12d98be41..64cf493028ad1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java
@@ -169,7 +169,7 @@ private InferenceState restoreInferenceState() {
         );
         try {
             Max maxIncrementalIdAgg = searchResponse.getAggregations().get(DestinationIndex.INCREMENTAL_ID);
-            long processedTestDocCount = searchResponse.getHits().getTotalHits().value;
+            long processedTestDocCount = searchResponse.getHits().getTotalHits().value();
             Long lastIncrementalId = processedTestDocCount == 0 ? null : (long) maxIncrementalIdAgg.value();
             if (lastIncrementalId != null) {
                 LOGGER.debug(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java
index 482e82f9ec303..fdd4bdd120f6a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java
@@ -115,7 +115,7 @@ private void searchIfTestDocsExist(ActionListener<Boolean> listener) {
             ML_ORIGIN,
             TransportSearchAction.TYPE,
             searchRequest,
-            listener.delegateFailureAndWrap((l, searchResponse) -> l.onResponse(searchResponse.getHits().getTotalHits().value > 0))
+            listener.delegateFailureAndWrap((l, searchResponse) -> l.onResponse(searchResponse.getHits().getTotalHits().value() > 0))
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/TrainTestSplitterFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/TrainTestSplitterFactory.java
index 3ef2affa5d399..0b3dd573deaae 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/TrainTestSplitterFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/TrainTestSplitterFactory.java
@@ -70,7 +70,7 @@ private TrainTestSplitter createSingleClassSplitter(Regression regression) {
                     regression.getDependentVariable(),
                     regression.getTrainingPercent(),
                     regression.getRandomizeSeed(),
-                    searchResponse.getHits().getTotalHits().value
+                    searchResponse.getHits().getTotalHits().value()
                 );
             } finally {
                 searchResponse.decRef();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/InferenceWaitForAllocation.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/InferenceWaitForAllocation.java
new file mode 100644
index 0000000000000..1142257a087c6
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/InferenceWaitForAllocation.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.inference;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.xpack.core.ml.action.InferModelAction;
+import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo;
+import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.core.Strings.format;
+
+/**
+ * Class for storing inference requests for ml trained models while
+ * scaling is in progress. Once the trained model has at least 1
+ * allocation the stored requests are forwarded to a consumer for
+ * processing.Requests will timeout while waiting for scale.
+ */
+public class InferenceWaitForAllocation {
+
+    public static final int MAX_PENDING_REQUEST_COUNT = 100;
+
+    /**
+     * Track details of the pending request
+     */
+    public record WaitingRequest(
+        InferModelAction.Request request,
+        InferModelAction.Response.Builder responseBuilder,
+        TaskId parentTaskId,
+        ActionListener<InferModelAction.Response> listener
+    ) {
+        public String deploymentId() {
+            return request.getId();
+        }
+    }
+
+    private static final Logger logger = LogManager.getLogger(InferenceWaitForAllocation.class);
+
+    private final TrainedModelAssignmentService assignmentService;
+    private final BiConsumer<WaitingRequest, TrainedModelAssignment> queuedConsumer;
+    private AtomicInteger pendingRequestCount = new AtomicInteger();
+
+    /**
+     * Create with consumer of the successful requests
+     * @param assignmentService            Trained model assignment service
+     * @param onInferenceScaledConsumer    The consumer of the waiting request called once an
+     *                                     allocation is available.
+     */
+    public InferenceWaitForAllocation(
+        TrainedModelAssignmentService assignmentService,
+        BiConsumer<WaitingRequest, TrainedModelAssignment> onInferenceScaledConsumer
+    ) {
+        this.assignmentService = assignmentService;
+        this.queuedConsumer = onInferenceScaledConsumer;
+    }
+
+    /**
+     * Wait for at least 1 allocation to be started then process the
+     * inference request.
+     * If the pending request count is greater than {@link #MAX_PENDING_REQUEST_COUNT}
+     * the request listener is failed with a too many requests exception
+     * The timeout is the inference request timeout.
+     * @param request The inference request details
+     */
+    public synchronized void waitForAssignment(WaitingRequest request) {
+        if (pendingRequestCount.incrementAndGet() >= MAX_PENDING_REQUEST_COUNT) {
+            pendingRequestCount.decrementAndGet();
+            request.listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Rejected inference request waiting for an allocation of deployment [{}]. Too many pending requests",
+                    RestStatus.TOO_MANY_REQUESTS,
+                    request.request.getId()
+                )
+            );
+            return;
+        }
+
+        var predicate = new DeploymentHasAtLeastOneAllocation(request.deploymentId());
+
+        assignmentService.waitForAssignmentCondition(
+            request.deploymentId(),
+            predicate,
+            request.request().getInferenceTimeout(),
+            new WaitingListener(request, predicate)
+        );
+    }
+
+    private static class DeploymentHasAtLeastOneAllocation implements Predicate<ClusterState> {
+
+        private final String deploymentId;
+        private AtomicReference<Exception> exception = new AtomicReference<>();
+
+        DeploymentHasAtLeastOneAllocation(String deploymentId) {
+            this.deploymentId = ExceptionsHelper.requireNonNull(deploymentId, "deployment_id");
+        }
+
+        @Override
+        public boolean test(ClusterState clusterState) {
+            TrainedModelAssignment trainedModelAssignment = TrainedModelAssignmentMetadata.assignmentForDeploymentId(
+                clusterState,
+                deploymentId
+            ).orElse(null);
+            if (trainedModelAssignment == null) {
+                logger.info(() -> format("[%s] assignment was null while waiting to scale up", deploymentId));
+                exception.set(
+                    new ElasticsearchStatusException(
+                        "[{}] Error waiting for a model allocation, model assignment has been removed",
+                        RestStatus.CONFLICT,
+                        deploymentId
+                    )
+                );
+                return true; // don't try again
+            }
+
+            Map<String, String> nodeFailuresAndReasons = new HashMap<>();
+            for (var nodeIdAndRouting : trainedModelAssignment.getNodeRoutingTable().entrySet()) {
+                if (RoutingState.FAILED.equals(nodeIdAndRouting.getValue().getState())) {
+                    nodeFailuresAndReasons.put(nodeIdAndRouting.getKey(), nodeIdAndRouting.getValue().getReason());
+                }
+            }
+            if (nodeFailuresAndReasons.isEmpty() == false) {
+                if (nodeFailuresAndReasons.size() == trainedModelAssignment.getNodeRoutingTable().size()) {
+                    exception.set(
+                        new ElasticsearchStatusException(
+                            "[{}] Error waiting for a model allocation, all nodes have failed with errors [{}]",
+                            RestStatus.INTERNAL_SERVER_ERROR,
+                            trainedModelAssignment.getDeploymentId(),
+                            nodeFailuresAndReasons
+                        )
+                    );
+                    return true; // don't try again
+                } else {
+                    logger.warn("Deployment [{}] has failed routes [{}]", trainedModelAssignment.getDeploymentId(), nodeFailuresAndReasons);
+                }
+            }
+
+            var routable = trainedModelAssignment.getNodeRoutingTable().values().stream().filter(RoutingInfo::isRoutable).findFirst();
+            return routable.isPresent();
+        }
+    }
+
+    private class WaitingListener implements TrainedModelAssignmentService.WaitForAssignmentListener {
+
+        private final WaitingRequest request;
+        private final DeploymentHasAtLeastOneAllocation predicate;
+
+        private WaitingListener(WaitingRequest request, DeploymentHasAtLeastOneAllocation predicate) {
+            this.request = request;
+            this.predicate = predicate;
+        }
+
+        @Override
+        public void onResponse(TrainedModelAssignment assignment) {
+            // assignment is started, do inference
+            pendingRequestCount.decrementAndGet();
+
+            if (predicate.exception.get() != null) {
+                onFailure(predicate.exception.get());
+                return;
+            }
+
+            queuedConsumer.accept(request, assignment);
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            pendingRequestCount.decrementAndGet();
+            request.listener().onFailure(e);
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java
index 58259b87c6b00..0dec99a9b9bb9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java
@@ -10,7 +10,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.TimeValue;
 
 /**
  * Processes measured requests counts and inference times and decides whether
@@ -22,12 +21,6 @@ public class AdaptiveAllocationsScaler {
     static final double SCALE_UP_THRESHOLD = 0.9;
     private static final double SCALE_DOWN_THRESHOLD = 0.85;
 
-    /**
-     * The time interval without any requests that has to pass, before scaling down
-     * to zero allocations (in case min_allocations = 0).
-     */
-    private static final long SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS = TimeValue.timeValueMinutes(15).getSeconds();
-
     /**
      * If the max_number_of_allocations is not set, use this value for now to prevent scaling up
      * to high numbers due to possible bugs or unexpected behaviour in the scaler.
@@ -40,6 +33,7 @@ public class AdaptiveAllocationsScaler {
     private final String deploymentId;
     private final KalmanFilter1d requestRateEstimator;
     private final KalmanFilter1d inferenceTimeEstimator;
+    private final long scaleToZeroAfterNoRequestsSeconds;
     private double timeWithoutRequestsSeconds;
 
     private int numberOfAllocations;
@@ -52,8 +46,10 @@ public class AdaptiveAllocationsScaler {
     private Double lastMeasuredInferenceTime;
     private Long lastMeasuredQueueSize;
 
-    AdaptiveAllocationsScaler(String deploymentId, int numberOfAllocations) {
+    AdaptiveAllocationsScaler(String deploymentId, int numberOfAllocations, long scaleToZeroAfterNoRequestsSeconds) {
         this.deploymentId = deploymentId;
+        this.scaleToZeroAfterNoRequestsSeconds = scaleToZeroAfterNoRequestsSeconds;
+
         // A smoothing factor of 100 roughly means the last 100 measurements have an effect
         // on the estimated values. The sampling time is 10 seconds, so approximately the
         // last 15 minutes are taken into account.
@@ -122,6 +118,10 @@ void process(AdaptiveAllocationsScalerService.Stats stats, double timeIntervalSe
         dynamicsChanged = false;
     }
 
+    void resetTimeWithoutRequests() {
+        timeWithoutRequestsSeconds = 0;
+    }
+
     double getLoadLower() {
         double requestRateLower = Math.max(0.0, requestRateEstimator.lower());
         double inferenceTimeLower = Math.max(0.0, inferenceTimeEstimator.hasValue() ? inferenceTimeEstimator.lower() : 1.0);
@@ -143,6 +143,7 @@ Double getInferenceTimeEstimate() {
     }
 
     Integer scale() {
+
         if (requestRateEstimator.hasValue() == false) {
             return null;
         }
@@ -170,9 +171,14 @@ Integer scale() {
         if (maxNumberOfAllocations != null) {
             numberOfAllocations = Math.min(numberOfAllocations, maxNumberOfAllocations);
         }
+
         if ((minNumberOfAllocations == null || minNumberOfAllocations == 0)
-            && timeWithoutRequestsSeconds > SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS) {
-            logger.debug("[{}] adaptive allocations scaler: scaling down to zero, because of no requests.", deploymentId);
+            && timeWithoutRequestsSeconds > scaleToZeroAfterNoRequestsSeconds) {
+
+            if (oldNumberOfAllocations != 0) {
+                // avoid logging this message if there is no change
+                logger.debug("[{}] adaptive allocations scaler: scaling down to zero, because of no requests.", deploymentId);
+            }
             numberOfAllocations = 0;
             neededNumberOfAllocations = 0;
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java
index 193fa9e7e07f9..16ec3ee9b468c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction;
 import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats;
@@ -40,6 +41,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Function;
 
@@ -171,17 +174,6 @@ Collection<DoubleWithAttributes> observeDouble(Function<AdaptiveAllocationsScale
             }
             return observations;
         }
-
-        void close() {
-            for (AutoCloseable metric : metrics) {
-                try {
-                    metric.close();
-                } catch (Exception e) {
-                    // do nothing
-                }
-            }
-            metrics.clear();
-        }
     }
 
     /**
@@ -194,6 +186,15 @@ void close() {
      */
     private static final long SCALE_UP_COOLDOWN_TIME_MILLIS = TimeValue.timeValueMinutes(5).getMillis();
 
+    /**
+     * The time interval without any requests that has to pass, before scaling down
+     * to zero allocations (in case min_allocations = 0). After this time interval
+     * without requests, the number of allocations is set to zero. When this time
+     * interval hasn't passed, the minimum number of allocations will always be
+     * larger than zero.
+     */
+    private static final long SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS = TimeValue.timeValueMinutes(15).getSeconds();
+
     private static final Logger logger = LogManager.getLogger(AdaptiveAllocationsScalerService.class);
 
     private final int timeIntervalSeconds;
@@ -208,9 +209,11 @@ void close() {
     private Long lastInferenceStatsTimestampMillis;
     private final Map<String, AdaptiveAllocationsScaler> scalers;
     private final Map<String, Long> lastScaleUpTimesMillis;
-
     private volatile Scheduler.Cancellable cancellable;
     private final AtomicBoolean busy;
+    private final long scaleToZeroAfterNoRequestsSeconds;
+    private final Set<String> deploymentIdsWithInFlightScaleFromZeroRequests = new ConcurrentSkipListSet<>();
+    private final Map<String, String> lastWarningMessages = new ConcurrentHashMap<>();
 
     public AdaptiveAllocationsScalerService(
         ThreadPool threadPool,
@@ -247,6 +250,7 @@ public AdaptiveAllocationsScalerService(
         scalers = new HashMap<>();
         metrics = new Metrics();
         busy = new AtomicBoolean(false);
+        scaleToZeroAfterNoRequestsSeconds = SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS;
     }
 
     public synchronized void start() {
@@ -261,7 +265,7 @@ public synchronized void start() {
     public synchronized void stop() {
         clusterService.removeListener(this);
         stopScheduling();
-        metrics.close();
+        scalers.clear();
     }
 
     @Override
@@ -290,7 +294,11 @@ private synchronized void updateAutoscalers(ClusterState state) {
                 && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) {
                 AdaptiveAllocationsScaler adaptiveAllocationsScaler = scalers.computeIfAbsent(
                     assignment.getDeploymentId(),
-                    key -> new AdaptiveAllocationsScaler(assignment.getDeploymentId(), assignment.totalTargetAllocations())
+                    key -> new AdaptiveAllocationsScaler(
+                        assignment.getDeploymentId(),
+                        assignment.totalTargetAllocations(),
+                        scaleToZeroAfterNoRequestsSeconds
+                    )
                 );
                 adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(
                     assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations(),
@@ -415,22 +423,49 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo
                 if (newNumberOfAllocations > numberOfAllocations.get(deploymentId)) {
                     lastScaleUpTimesMillis.put(deploymentId, now);
                 }
-                updateNumberOfAllocations(deploymentId, newNumberOfAllocations);
+                updateNumberOfAllocations(
+                    deploymentId,
+                    newNumberOfAllocations,
+                    updateAssigmentListener(deploymentId, newNumberOfAllocations)
+                );
             }
         }
     }
 
     public boolean maybeStartAllocation(TrainedModelAssignment assignment) {
         if (assignment.getAdaptiveAllocationsSettings() != null
-            && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) {
-            lastScaleUpTimesMillis.put(assignment.getDeploymentId(), System.currentTimeMillis());
-            updateNumberOfAllocations(assignment.getDeploymentId(), 1);
+            && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE
+            && (assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations() == null
+                || assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations() == 0)) {
+
+            // Prevent against a flurry of scale up requests.
+            if (deploymentIdsWithInFlightScaleFromZeroRequests.contains(assignment.getDeploymentId()) == false) {
+                lastScaleUpTimesMillis.put(assignment.getDeploymentId(), System.currentTimeMillis());
+                var updateListener = updateAssigmentListener(assignment.getDeploymentId(), 1);
+                var cleanUpListener = ActionListener.runAfter(
+                    updateListener,
+                    () -> deploymentIdsWithInFlightScaleFromZeroRequests.remove(assignment.getDeploymentId())
+                );
+
+                deploymentIdsWithInFlightScaleFromZeroRequests.add(assignment.getDeploymentId());
+                updateNumberOfAllocations(assignment.getDeploymentId(), 1, cleanUpListener);
+            }
+
+            AdaptiveAllocationsScaler scaler = scalers.get(assignment.getDeploymentId());
+            if (scaler != null) {
+                scaler.resetTimeWithoutRequests();
+            }
+
             return true;
         }
         return false;
     }
 
-    private void updateNumberOfAllocations(String deploymentId, int numberOfAllocations) {
+    private void updateNumberOfAllocations(
+        String deploymentId,
+        int numberOfAllocations,
+        ActionListener<CreateTrainedModelAssignmentAction.Response> listener
+    ) {
         UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(deploymentId);
         updateRequest.setNumberOfAllocations(numberOfAllocations);
         updateRequest.setIsInternal(true);
@@ -439,23 +474,31 @@ private void updateNumberOfAllocations(String deploymentId, int numberOfAllocati
             ClientHelper.ML_ORIGIN,
             UpdateTrainedModelDeploymentAction.INSTANCE,
             updateRequest,
-            ActionListener.wrap(updateResponse -> {
-                logger.info("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, numberOfAllocations);
-                threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
-                    .execute(
-                        () -> inferenceAuditor.info(
-                            deploymentId,
-                            Strings.format(
-                                "adaptive allocations scaler: scaled [%s] to [%s] allocations.",
-                                deploymentId,
-                                numberOfAllocations
-                            )
-                        )
-                    );
-            }, e -> {
-                logger.atLevel(Level.WARN)
-                    .withThrowable(e)
-                    .log("adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", deploymentId, numberOfAllocations);
+            listener
+        );
+    }
+
+    private ActionListener<CreateTrainedModelAssignmentAction.Response> updateAssigmentListener(
+        String deploymentId,
+        int numberOfAllocations
+    ) {
+        return ActionListener.wrap(updateResponse -> {
+            lastWarningMessages.remove(deploymentId);
+            logger.info("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, numberOfAllocations);
+            threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
+                .execute(
+                    () -> inferenceAuditor.info(
+                        deploymentId,
+                        Strings.format("adaptive allocations scaler: scaled [%s] to [%s] allocations.", deploymentId, numberOfAllocations)
+                    )
+                );
+        }, e -> {
+            Level level = e.getMessage().equals(lastWarningMessages.get(deploymentId)) ? Level.DEBUG : Level.WARN;
+            lastWarningMessages.put(deploymentId, e.getMessage());
+            logger.atLevel(level)
+                .withThrowable(e)
+                .log("adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", deploymentId, numberOfAllocations);
+            if (level == Level.WARN) {
                 threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
                     .execute(
                         () -> inferenceAuditor.warning(
@@ -467,7 +510,7 @@ private void updateNumberOfAllocations(String deploymentId, int numberOfAllocati
                             )
                         )
                     );
-            })
-        );
+            }
+        });
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleFromZeroFeatureFlag.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleFromZeroFeatureFlag.java
new file mode 100644
index 0000000000000..4c446b65db9dd
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleFromZeroFeatureFlag.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.inference.adaptiveallocations;
+
+import org.elasticsearch.common.util.FeatureFlag;
+
+public class ScaleFromZeroFeatureFlag {
+    private ScaleFromZeroFeatureFlag() {}
+
+    private static final FeatureFlag FEATURE_FLAG = new FeatureFlag("ml_scale_from_zero");
+
+    public static boolean isEnabled() {
+        return FEATURE_FLAG.isEnabled();
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
index 96490716c5c6c..0439ef9db30b4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
@@ -113,7 +113,7 @@ public TrainedModelAssignmentClusterService(
         this.maxMemoryPercentage = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings);
         this.useAuto = MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT.get(settings);
         this.maxOpenJobs = MachineLearning.MAX_OPEN_JOBS_PER_NODE.get(settings);
-        this.maxLazyMLNodes = MachineLearning.MAX_LAZY_ML_NODES.get(settings);
+        this.maxLazyMLNodes = MachineLearningField.MAX_LAZY_ML_NODES.get(settings);
         this.maxMLNodeSize = MachineLearning.MAX_ML_NODE_SIZE.get(settings).getBytes();
         this.allocatedProcessorsScale = MachineLearning.ALLOCATED_PROCESSORS_SCALE.get(settings);
         this.client = client;
@@ -125,7 +125,7 @@ public TrainedModelAssignmentClusterService(
             clusterService.getClusterSettings()
                 .addSettingsUpdateConsumer(MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, this::setUseAuto);
             clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_OPEN_JOBS_PER_NODE, this::setMaxOpenJobs);
-            clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes);
+            clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearningField.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes);
             clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_ML_NODE_SIZE, this::setMaxMLNodeSize);
             clusterService.getClusterSettings()
                 .addSettingsUpdateConsumer(MachineLearning.ALLOCATED_PROCESSORS_SCALE, this::setAllocatedProcessorsScale);
@@ -845,10 +845,9 @@ private void updateDeployment(
                 return;
             }
         }
-        boolean hasUpdates = (numberOfAllocations != null
-            && Objects.equals(numberOfAllocations, existingAssignment.getTaskParams().getNumberOfAllocations()) == false)
-            || Objects.equals(adaptiveAllocationsSettings, existingAssignment.getAdaptiveAllocationsSettings()) == false;
+        boolean hasUpdates = hasUpdates(numberOfAllocations, adaptiveAllocationsSettingsUpdates, existingAssignment);
         if (hasUpdates == false) {
+            logger.info("no updates");
             listener.onResponse(existingAssignment);
             return;
         }
@@ -917,6 +916,17 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState)
         updateAssignment(clusterState, existingAssignment, numberOfAllocations, adaptiveAllocationsSettings, updatedStateListener);
     }
 
+    static boolean hasUpdates(
+        Integer proposedNumberOfAllocations,
+        AdaptiveAllocationsSettings proposedAdaptiveSettings,
+        TrainedModelAssignment existingAssignment
+    ) {
+        return (proposedNumberOfAllocations != null
+            && Objects.equals(proposedNumberOfAllocations, existingAssignment.getTaskParams().getNumberOfAllocations()) == false)
+            || (proposedAdaptiveSettings != null
+                && Objects.equals(proposedAdaptiveSettings, existingAssignment.getAdaptiveAllocationsSettings()) == false);
+    }
+
     private AdaptiveAllocationsSettings getAdaptiveAllocationsSettings(
         AdaptiveAllocationsSettings original,
         AdaptiveAllocationsSettings updates
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
index 70d0b980bb3bf..54a9fe908fa87 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
@@ -35,6 +35,7 @@
 
 public class LearningToRankRescorer implements Rescorer {
 
+    private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10;
     public static final LearningToRankRescorer INSTANCE = new LearningToRankRescorer();
     private static final Logger logger = LogManager.getLogger(LearningToRankRescorer.class);
 
@@ -78,7 +79,12 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
         List<FeatureExtractor> featureExtractors = ltrRescoreContext.buildFeatureExtractors(searcher);
         List<Map<String, Object>> docFeatures = new ArrayList<>(topDocIDs.size());
         int featureSize = featureExtractors.stream().mapToInt(fe -> fe.featureNames().size()).sum();
+        int count = 0;
         while (hitUpto < hitsToRescore.length) {
+            if (count % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) {
+                rescoreContext.checkCancellation();
+            }
+            count++;
             final ScoreDoc hit = hitsToRescore[hitUpto];
             final int docID = hit.doc;
             while (docID >= endDoc) {
@@ -106,6 +112,9 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
             hitUpto++;
         }
         for (int i = 0; i < hitsToRescore.length; i++) {
+            if (i % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) {
+                rescoreContext.checkCancellation();
+            }
             Map<String, Object> features = docFeatures.get(i);
             try {
                 InferenceResults results = definition.inferLtr(features, ltrRescoreContext.learningToRankConfig);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java
index f56c589aea19a..c4396c4f9d2c8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java
@@ -196,7 +196,7 @@ private void doSearch(
                 numDocsWritten += searchResponse.getHits().getHits().length;
 
                 boolean endOfSearch = searchResponse.getHits().getHits().length < searchSize
-                    || searchResponse.getHits().getTotalHits().value == numDocsWritten;
+                    || searchResponse.getHits().getTotalHits().value() == numDocsWritten;
 
                 if (endOfSearch) {
                     successConsumer.accept(Boolean.TRUE);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java
index f493c735d87ea..ff5f37427b18f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java
@@ -1008,7 +1008,7 @@ public void expandIds(
             ML_ORIGIN,
             searchRequest,
             ActionListener.<SearchResponse>wrap(response -> {
-                long totalHitCount = response.getHits().getTotalHits().value + foundResourceIds.size();
+                long totalHitCount = response.getHits().getTotalHits().value() + foundResourceIds.size();
                 Set<String> foundFromDocs = new HashSet<>();
                 for (SearchHit hit : response.getHits().getHits()) {
                     Map<String, Object> docSource = hit.getSourceAsMap();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
index 8493513f40bd6..df9a187f59616 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
@@ -683,7 +683,7 @@ public void groupExists(String groupId, ActionListener<Boolean> listener) {
             ML_ORIGIN,
             searchRequest,
             ActionListener.<SearchResponse>wrap(
-                response -> listener.onResponse(response.getHits().getTotalHits().value > 0),
+                response -> listener.onResponse(response.getHits().getTotalHits().value() > 0),
                 listener::onFailure
             ),
             client::search
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
index b9cc1902b7ab6..0f3abe3ab8c20 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
@@ -352,7 +352,7 @@ public void deleteJobDocuments(
                     }
                 }
                 SearchResponse searchResponse = item.getResponse();
-                if (searchResponse.getHits().getTotalHits().value > 0 || indexNames.get()[i].equals(defaultSharedIndex)) {
+                if (searchResponse.getHits().getTotalHits().value() > 0 || indexNames.get()[i].equals(defaultSharedIndex)) {
                     needToRunDBQTemp = true;
                 } else {
                     indicesToDelete.add(indexNames.get()[i]);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
index f9e4e62e4e3bc..51b3e0b55d75b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
@@ -870,7 +870,7 @@ public void buckets(
                     throw QueryPage.emptyQueryPage(Bucket.RESULTS_FIELD);
                 }
 
-                QueryPage<Bucket> buckets = new QueryPage<>(results, searchResponse.getHits().getTotalHits().value, Bucket.RESULTS_FIELD);
+                QueryPage<Bucket> buckets = new QueryPage<>(results, searchResponse.getHits().getTotalHits().value(), Bucket.RESULTS_FIELD);
 
                 if (query.isExpand()) {
                     Iterator<Bucket> bucketsToExpand = buckets.results()
@@ -1086,7 +1086,7 @@ public void categoryDefinitions(
                 }
                 QueryPage<CategoryDefinition> result = new QueryPage<>(
                     results,
-                    searchResponse.getHits().getTotalHits().value,
+                    searchResponse.getHits().getTotalHits().value(),
                     CategoryDefinition.RESULTS_FIELD
                 );
                 handler.accept(result);
@@ -1143,7 +1143,7 @@ public void records(
                 }
                 QueryPage<AnomalyRecord> queryPage = new QueryPage<>(
                     results,
-                    searchResponse.getHits().getTotalHits().value,
+                    searchResponse.getHits().getTotalHits().value(),
                     AnomalyRecord.RESULTS_FIELD
                 );
                 handler.accept(queryPage);
@@ -1207,7 +1207,7 @@ public void influencers(
                 }
                 QueryPage<Influencer> result = new QueryPage<>(
                     influencers,
-                    response.getHits().getTotalHits().value,
+                    response.getHits().getTotalHits().value(),
                     Influencer.RESULTS_FIELD
                 );
                 handler.accept(result);
@@ -1375,7 +1375,7 @@ private void modelSnapshots(
 
                 QueryPage<ModelSnapshot> result = new QueryPage<>(
                     results,
-                    searchResponse.getHits().getTotalHits().value,
+                    searchResponse.getHits().getTotalHits().value(),
                     ModelSnapshot.RESULTS_FIELD
                 );
                 handler.accept(result);
@@ -1411,7 +1411,7 @@ public QueryPage<ModelPlot> modelPlot(String jobId, int from, int size) {
                 }
             }
 
-            return new QueryPage<>(results, searchResponse.getHits().getTotalHits().value, ModelPlot.RESULTS_FIELD);
+            return new QueryPage<>(results, searchResponse.getHits().getTotalHits().value(), ModelPlot.RESULTS_FIELD);
         } finally {
             searchResponse.decRef();
         }
@@ -1444,7 +1444,7 @@ public QueryPage<CategorizerStats> categorizerStats(String jobId, int from, int
                 }
             }
 
-            return new QueryPage<>(results, searchResponse.getHits().getTotalHits().value, ModelPlot.RESULTS_FIELD);
+            return new QueryPage<>(results, searchResponse.getHits().getTotalHits().value(), ModelPlot.RESULTS_FIELD);
         } finally {
             searchResponse.decRef();
         }
@@ -1700,7 +1700,7 @@ public void scheduledEvents(ScheduledEventsQueryBuilder query, ActionListener<Qu
                         event.eventId(hit.getId());
                         events.add(event.build());
                     }
-                    handler.onResponse(new QueryPage<>(events, response.getHits().getTotalHits().value, ScheduledEvent.RESULTS_FIELD));
+                    handler.onResponse(new QueryPage<>(events, response.getHits().getTotalHits().value(), ScheduledEvent.RESULTS_FIELD));
                 } catch (Exception e) {
                     handler.onFailure(e);
                 }
@@ -1901,7 +1901,7 @@ public void calendars(CalendarQueryBuilder queryBuilder, ActionListener<QueryPag
                     for (SearchHit hit : hits) {
                         calendars.add(MlParserUtils.parse(hit, Calendar.LENIENT_PARSER).build());
                     }
-                    listener.onResponse(new QueryPage<>(calendars, response.getHits().getTotalHits().value, Calendar.RESULTS_FIELD));
+                    listener.onResponse(new QueryPage<>(calendars, response.getHits().getTotalHits().value(), Calendar.RESULTS_FIELD));
                 } catch (Exception e) {
                     listener.onFailure(e);
                 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
index 886c19a65a4d0..194759c026a30 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
@@ -168,7 +168,7 @@ private List<JobForecastId> findForecastsToDelete(SearchResponse searchResponse)
         List<JobForecastId> forecastsToDelete = new ArrayList<>();
 
         SearchHits hits = searchResponse.getHits();
-        if (hits.getTotalHits().value > MAX_FORECASTS) {
+        if (hits.getTotalHits().value() > MAX_FORECASTS) {
             LOGGER.info("More than [{}] forecasts were found. This run will only delete [{}] of them", MAX_FORECASTS, MAX_FORECASTS);
         }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilder.java
index cb858940b64d1..5a63ad8e85e9b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilder.java
@@ -310,7 +310,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.SPARSE_VECTOR_QUERY_ADDED;
+        return TransportVersions.V_8_15_0;
     }
 
     private static final ConstructingObjectParser<SparseVectorQueryBuilder, Void> PARSER = new ConstructingObjectParser<>(NAME, a -> {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestDeleteExpiredDataAction.java
index 37731fcbfb10b..efa34ccff70c4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestDeleteExpiredDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestDeleteExpiredDataAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -20,8 +19,8 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestDeleteExpiredDataAction extends BaseRestHandler {
@@ -29,12 +28,8 @@ public class RestDeleteExpiredDataAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(DELETE, BASE_PATH + "_delete_expired_data/{" + Job.ID + "}")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "_delete_expired_data/{" + Job.ID + "}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(DELETE, BASE_PATH + "_delete_expired_data")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "_delete_expired_data", RestApiVersion.V_7)
-                .build()
+            new Route(DELETE, BASE_PATH + "_delete_expired_data/{" + ID + "}"),
+            new Route(DELETE, BASE_PATH + "_delete_expired_data")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java
index aace51a45c77b..d08f25b3433b5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -20,14 +19,13 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestMlInfoAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(GET, BASE_PATH + "info").replaces(GET, PRE_V7_BASE_PATH + "info", RestApiVersion.V_7).build());
+        return List.of(new Route(GET, BASE_PATH + "info"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestSetUpgradeModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestSetUpgradeModeAction.java
index 67970d2ad056e..1d12a8b555225 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestSetUpgradeModeAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestSetUpgradeModeAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -22,18 +21,13 @@
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestSetUpgradeModeAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "set_upgrade_mode")
-                .replaces(POST, PRE_V7_BASE_PATH + "set_upgrade_mode", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "set_upgrade_mode"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java
index 2825d1e4433af..bed1ebc370520 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.calendar;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -20,19 +19,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteCalendarAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, BASE_PATH + "calendars/{" + Calendar.ID + "}")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, BASE_PATH + "calendars/{" + ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarEventAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarEventAction.java
index ec23eb8ea3551..f645d6cb7efbd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarEventAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarEventAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.calendar;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,23 +20,16 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
+import static org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent.EVENT_ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteCalendarEventAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, BASE_PATH + "calendars/{" + Calendar.ID + "}/events/{" + ScheduledEvent.EVENT_ID + "}")
-                .replaces(
-                    DELETE,
-                    PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}/events/{" + ScheduledEvent.EVENT_ID + "}",
-                    RestApiVersion.V_7
-                )
-                .build()
-        );
+        return List.of(new Route(DELETE, BASE_PATH + "calendars/{" + ID + "}/events/{" + EVENT_ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java
index 342a9008a69b8..782fdc9073ab4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.calendar;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,19 +20,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteCalendarJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, BASE_PATH + "calendars/{" + Calendar.ID + "}/jobs/{" + Job.ID + "}")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}/jobs/{" + Job.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, BASE_PATH + "calendars/{" + ID + "}/jobs/{" + Job.ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarEventsAction.java
index be5f9cd3c1446..4bf2cfa8472fe 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarEventsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarEventsAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.calendar;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,19 +22,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetCalendarEventsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, BASE_PATH + "calendars/{" + Calendar.ID + "}/events")
-                .replaces(GET, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}/events", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, BASE_PATH + "calendars/{" + ID + "}/events"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarsAction.java
index 1e662e28cf564..16f2a840b4575 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestGetCalendarsAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -24,8 +23,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetCalendarsAction extends BaseRestHandler {
@@ -33,14 +32,10 @@ public class RestGetCalendarsAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "calendars/{" + Calendar.ID + "}")
-                .replaces(GET, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, BASE_PATH + "calendars/").replaces(GET, PRE_V7_BASE_PATH + "calendars/", RestApiVersion.V_7).build(),
-            Route.builder(POST, BASE_PATH + "calendars/{" + Calendar.ID + "}")
-                .replaces(POST, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "calendars/").replaces(POST, PRE_V7_BASE_PATH + "calendars/", RestApiVersion.V_7).build()
+            new Route(GET, BASE_PATH + "calendars/{" + ID + "}"),
+            new Route(GET, BASE_PATH + "calendars/"),
+            new Route(POST, BASE_PATH + "calendars/{" + ID + "}"),
+            new Route(POST, BASE_PATH + "calendars/")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPostCalendarEventAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPostCalendarEventAction.java
index 9c686e441f4e3..525127cb0fde4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPostCalendarEventAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPostCalendarEventAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.calendar;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,19 +20,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPostCalendarEventAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "calendars/{" + Calendar.ID + "}/events")
-                .replaces(POST, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}/events", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "calendars/{" + ID + "}/events"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarAction.java
index 6b4b709de6ada..3457195bf1499 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.calendar;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -22,19 +21,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutCalendarAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(PUT, BASE_PATH + "calendars/{" + Calendar.ID + "}")
-                .replaces(PUT, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(PUT, BASE_PATH + "calendars/{" + ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java
index bdbda6d931f73..694100045aa8a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.calendar;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,19 +20,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
+import static org.elasticsearch.xpack.core.ml.calendars.Calendar.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutCalendarJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(PUT, BASE_PATH + "calendars/{" + Calendar.ID + "}/jobs/{" + Job.ID + "}")
-                .replaces(PUT, PRE_V7_BASE_PATH + "calendars/{" + Calendar.ID + "}/jobs/{" + Job.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(PUT, BASE_PATH + "calendars/{" + ID + "}/jobs/{" + Job.ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java
index b93089eabda2d..205bb4f68a62c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.Table;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.Scope;
@@ -51,6 +52,7 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c
         if (Strings.isNullOrEmpty(datafeedId)) {
             datafeedId = GetDatafeedsStatsAction.ALL;
         }
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request = new Request(datafeedId);
         checkAndSetDeprecatedParam(
             DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java
index cb02990da74c9..b27819bceee44 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.Scope;
@@ -56,6 +57,7 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c
         if (Strings.isNullOrEmpty(jobId)) {
             jobId = Metadata.ALL;
         }
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request = new Request(jobId);
         checkAndSetDeprecatedParam(
             DEPRECATED_ALLOW_NO_JOBS_PARAM,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestDeleteDatafeedAction.java
index c195da8fbb0f4..6b4777e112254 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestDeleteDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestDeleteDatafeedAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.datafeeds;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,19 +22,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteDatafeedAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, BASE_PATH + "datafeeds/{" + ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java
index e676f7d0a07db..8c85c055fca3b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -24,8 +25,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -33,14 +34,7 @@ public class RestGetDatafeedStatsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_stats")
-                .replaces(GET, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_stats", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, BASE_PATH + "datafeeds/_stats")
-                .replaces(GET, PRE_V7_BASE_PATH + "datafeeds/_stats", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, BASE_PATH + "datafeeds/{" + ID + "}/_stats"), new Route(GET, BASE_PATH + "datafeeds/_stats"));
     }
 
     @Override
@@ -54,6 +48,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         if (Strings.isNullOrEmpty(datafeedId)) {
             datafeedId = GetDatafeedsStatsAction.ALL;
         }
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request = new Request(datafeedId);
         checkAndSetDeprecatedParam(
             DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java
index 53f0a371fb9d4..fd0681f68a3a5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -26,9 +27,9 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -36,12 +37,7 @@ public class RestGetDatafeedsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}")
-                .replaces(GET, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, BASE_PATH + "datafeeds").replaces(GET, PRE_V7_BASE_PATH + "datafeeds", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, BASE_PATH + "datafeeds/{" + ID + "}"), new Route(GET, BASE_PATH + "datafeeds"));
     }
 
     @Override
@@ -55,6 +51,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         if (datafeedId == null) {
             datafeedId = GetDatafeedsAction.ALL;
         }
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request = new Request(datafeedId);
         checkAndSetDeprecatedParam(
             DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java
index 999449b4fa206..f6a73f0cb8c84 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.datafeeds;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,8 +22,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPreviewDatafeedAction extends BaseRestHandler {
@@ -32,18 +31,10 @@ public class RestPreviewDatafeedAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview")
-                .replaces(GET, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, BASE_PATH + "datafeeds/_preview")
-                .replaces(GET, PRE_V7_BASE_PATH + "datafeeds/_preview", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview")
-                .replaces(POST, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "datafeeds/_preview")
-                .replaces(POST, PRE_V7_BASE_PATH + "datafeeds/_preview", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "datafeeds/{" + ID + "}/_preview"),
+            new Route(GET, BASE_PATH + "datafeeds/_preview"),
+            new Route(POST, BASE_PATH + "datafeeds/{" + ID + "}/_preview"),
+            new Route(POST, BASE_PATH + "datafeeds/_preview")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java
index 64caa8b737e3c..360b14277a19a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -25,19 +24,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutDatafeedAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(PUT, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}")
-                .replaces(PUT, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(PUT, BASE_PATH + "datafeeds/{" + ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java
index 8d6e6d5336a1f..82db97dced01e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.datafeeds;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -26,8 +25,8 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestStartDatafeedAction extends BaseRestHandler {
@@ -36,11 +35,7 @@ public class RestStartDatafeedAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_start")
-                .replaces(POST, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_start", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "datafeeds/{" + ID + "}/_start"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java
index 862c0b5318f29..8235e2785cc37 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -28,8 +29,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -37,11 +38,7 @@ public class RestStopDatafeedAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_stop")
-                .replaces(POST, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_stop", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "datafeeds/{" + ID + "}/_stop"));
     }
 
     @Override
@@ -52,6 +49,7 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
         String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName());
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request;
         if (restRequest.hasContentOrSourceParam()) {
             XContentParser parser = restRequest.contentOrSourceParamParser();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java
index 44a8415a91fde..5335de78aa18a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -25,19 +24,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestUpdateDatafeedAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_update")
-                .replaces(POST, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_update", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "datafeeds/{" + ID + "}/_update"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestDeleteFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestDeleteFilterAction.java
index 494c5638dc75a..57c99627b9330 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestDeleteFilterAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestDeleteFilterAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.filter;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -22,19 +21,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.ml.action.DeleteFilterAction.Request.FILTER_ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteFilterAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, BASE_PATH + "filters/{" + Request.FILTER_ID + "}")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "filters/{" + Request.FILTER_ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, BASE_PATH + "filters/{" + FILTER_ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestGetFiltersAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestGetFiltersAction.java
index b4c76c3fa4475..45a30796d56f3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestGetFiltersAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestGetFiltersAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,20 +22,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.core.ml.job.config.MlFilter.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetFiltersAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, BASE_PATH + "filters/{" + MlFilter.ID + "}")
-                .replaces(GET, PRE_V7_BASE_PATH + "filters/{" + MlFilter.ID + "}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, BASE_PATH + "filters/").replaces(GET, PRE_V7_BASE_PATH + "filters/", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, BASE_PATH + "filters/{" + ID + "}"), new Route(GET, BASE_PATH + "filters/"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestPutFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestPutFilterAction.java
index 78753a46dc492..1d62137666d93 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestPutFilterAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestPutFilterAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.filter;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,19 +20,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
+import static org.elasticsearch.xpack.core.ml.job.config.MlFilter.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutFilterAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(PUT, BASE_PATH + "filters/{" + MlFilter.ID + "}")
-                .replaces(PUT, PRE_V7_BASE_PATH + "filters/{" + MlFilter.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(PUT, BASE_PATH + "filters/{" + ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestUpdateFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestUpdateFilterAction.java
index a2cdb4a85395a..b32997d0eee45 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestUpdateFilterAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestUpdateFilterAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.filter;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,19 +20,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.MlFilter.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestUpdateFilterAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "filters/{" + MlFilter.ID + "}/_update")
-                .replaces(POST, PRE_V7_BASE_PATH + "filters/{" + MlFilter.ID + "}/_update", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "filters/{" + ID + "}/_update"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelDefinitionPartAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelDefinitionPartAction.java
index d328b90a8e0d3..fc2c102747992 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelDefinitionPartAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelDefinitionPartAction.java
@@ -20,6 +20,8 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
+import static org.elasticsearch.xpack.core.ml.action.PutTrainedModelDefinitionPartAction.Request.PART;
+import static org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig.MODEL_ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -27,17 +29,7 @@ public class RestPutTrainedModelDefinitionPartAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(
-                PUT,
-                BASE_PATH
-                    + "trained_models/{"
-                    + TrainedModelConfig.MODEL_ID.getPreferredName()
-                    + "}/definition/{"
-                    + PutTrainedModelDefinitionPartAction.Request.PART
-                    + "}"
-            ).build()
-        );
+        return List.of(new Route(PUT, BASE_PATH + "trained_models/{" + MODEL_ID.getPreferredName() + "}/definition/{" + PART + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelVocabularyAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelVocabularyAction.java
index ee228223eb340..34f6c4b9f51bb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelVocabularyAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelVocabularyAction.java
@@ -20,6 +20,7 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
+import static org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig.MODEL_ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -27,9 +28,7 @@ public class RestPutTrainedModelVocabularyAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(PUT, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID.getPreferredName() + "}/vocabulary").build()
-        );
+        return List.of(new Route(PUT, BASE_PATH + "trained_models/{" + MODEL_ID.getPreferredName() + "}/vocabulary"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java
index 6ee4ba8be376e..f98a2f5a933ae 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,8 +24,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -32,11 +33,7 @@ public class RestCloseJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_close")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_close", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/_close"));
     }
 
     @Override
@@ -46,6 +43,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request;
         if (restRequest.hasContentOrSourceParam()) {
             request = Request.parseRequest(restRequest.param(Job.ID.getPreferredName()), restRequest.contentParser());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteForecastAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteForecastAction.java
index ba3bf73a3a4d6..13cb58964733f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteForecastAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteForecastAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,8 +22,9 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
+import static org.elasticsearch.xpack.core.ml.job.results.Forecast.FORECAST_ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestDeleteForecastAction extends BaseRestHandler {
@@ -32,16 +32,8 @@ public class RestDeleteForecastAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(DELETE, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_forecast/")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_forecast/", RestApiVersion.V_7)
-                .build(),
-            Route.builder(DELETE, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_forecast/{" + Forecast.FORECAST_ID + "}")
-                .replaces(
-                    DELETE,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_forecast/{" + Forecast.FORECAST_ID + "}",
-                    RestApiVersion.V_7
-                )
-                .build()
+            new Route(DELETE, BASE_PATH + "anomaly_detectors/{" + ID + "}/_forecast/"),
+            new Route(DELETE, BASE_PATH + "anomaly_detectors/{" + ID + "}/_forecast/{" + FORECAST_ID + "}")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteJobAction.java
index 1db528cae4a3d..0a4e1c0ea68bb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteJobAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -28,19 +27,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}")
-                .replaces(DELETE, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, BASE_PATH + "anomaly_detectors/{" + ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestFlushJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestFlushJobAction.java
index d8784a369594c..650c5d92e7589 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestFlushJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestFlushJobAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.job;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,8 +20,8 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestFlushJobAction extends BaseRestHandler {
@@ -35,11 +34,7 @@ public class RestFlushJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_flush")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_flush", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/_flush"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java
index 5f3388c143dd4..b45fe5aeb89a0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -22,19 +21,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestForecastJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_forecast")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_forecast", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/_forecast"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java
index d77a913ab0491..2899faabdc40f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -25,8 +26,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -35,12 +36,8 @@ public class RestGetJobStatsAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_stats")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_stats", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/_stats")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/_stats", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/_stats"),
+            new Route(GET, BASE_PATH + "anomaly_detectors/_stats")
         );
     }
 
@@ -55,6 +52,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         if (Strings.isNullOrEmpty(jobId)) {
             jobId = Metadata.ALL;
         }
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request = new Request(jobId);
         checkAndSetDeprecatedParam(
             DEPRECATED_ALLOW_NO_JOBS_PARAM,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java
index f3c7b8c6636ef..ae8d234d1d8bd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -27,9 +28,9 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -37,14 +38,7 @@ public class RestGetJobsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, BASE_PATH + "anomaly_detectors")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}"), new Route(GET, BASE_PATH + "anomaly_detectors"));
     }
 
     @Override
@@ -58,6 +52,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         if (Strings.isNullOrEmpty(jobId)) {
             jobId = Metadata.ALL;
         }
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         Request request = new Request(jobId);
         checkAndSetDeprecatedParam(
             DEPRECATED_ALLOW_NO_JOBS_PARAM,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java
index dba1a7a2fb132..74dfc959acb26 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.job;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -25,19 +24,15 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestOpenJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_open")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_open", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/_open"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java
index 8c7c12de0ce51..48c6abde3010a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.job;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -21,7 +20,6 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 public class RestPostDataAction extends BaseRestHandler {
 
@@ -39,8 +37,7 @@ public List<Route> routes() {
             + "in a future major version it will be compulsory to use a datafeed";
         return List.of(
             // Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecated(msg, RestApiVersion.V_8).build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecateAndKeep(msg).build(),
-            Route.builder(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecated(msg, RestApiVersion.V_7).build()
+            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecateAndKeep(msg).build()
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostJobUpdateAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostJobUpdateAction.java
index 7b2097012e8c1..e1e8296a62073 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostJobUpdateAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostJobUpdateAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.job;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,19 +22,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPostJobUpdateAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_update")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_update", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/_update"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPutJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPutJobAction.java
index 4c754d9bcfdd9..743fe8fac4317 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPutJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPutJobAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -25,19 +24,15 @@
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(PUT, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}")
-                .replaces(PUT, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(PUT, BASE_PATH + "anomaly_detectors/{" + ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestDeleteModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestDeleteModelSnapshotAction.java
index d3069f4f6c405..2a9d83ccfc292 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestDeleteModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestDeleteModelSnapshotAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.modelsnapshots;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -20,24 +19,16 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField.SNAPSHOT_ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestDeleteModelSnapshotAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}")
-                .replaces(
-                    DELETE,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}",
-                    RestApiVersion.V_7
-                )
-                .build()
-        );
+        return List.of(new Route(DELETE, BASE_PATH + "anomaly_detectors/{" + ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestGetModelSnapshotsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestGetModelSnapshotsAction.java
index 7cca7d250d030..ec2d567700df8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestGetModelSnapshotsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestGetModelSnapshotsAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -26,8 +25,9 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction.Request.SNAPSHOT_ID;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestGetModelSnapshotsAction extends BaseRestHandler {
@@ -44,26 +44,10 @@ public class RestGetModelSnapshotsAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + Request.SNAPSHOT_ID + "}")
-                .replaces(
-                    GET,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + Request.SNAPSHOT_ID + "}",
-                    RestApiVersion.V_7
-                )
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + Request.SNAPSHOT_ID + "}")
-                .replaces(
-                    POST,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + Request.SNAPSHOT_ID + "}",
-                    RestApiVersion.V_7
-                )
-                .build(),
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}"),
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/model_snapshots"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/model_snapshots")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestRevertModelSnapshotAction.java
index 20833853b3107..c4a441d637fb0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestRevertModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestRevertModelSnapshotAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.modelsnapshots;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -24,8 +23,8 @@
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 import static org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction.Request.SNAPSHOT_ID;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestRevertModelSnapshotAction extends BaseRestHandler {
@@ -34,15 +33,7 @@ public class RestRevertModelSnapshotAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}/_revert")
-                .replaces(
-                    POST,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}/_revert",
-                    RestApiVersion.V_7
-                )
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}/_revert"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestUpdateModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestUpdateModelSnapshotAction.java
index 911305aca474c..e540d7115b3cd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestUpdateModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestUpdateModelSnapshotAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.modelsnapshots;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,24 +20,16 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField.SNAPSHOT_ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestUpdateModelSnapshotAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}/_update")
-                .replaces(
-                    POST,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}/_update",
-                    RestApiVersion.V_7
-                )
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/model_snapshots/{" + SNAPSHOT_ID + "}/_update"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java
index babffc3fd4650..a049529c889fc 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -18,15 +17,15 @@
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.ml.action.GetBucketsAction;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.job.results.Result;
 
 import java.io.IOException;
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
+import static org.elasticsearch.xpack.core.ml.job.results.Result.TIMESTAMP;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestGetBucketsAction extends BaseRestHandler {
@@ -34,26 +33,10 @@ public class RestGetBucketsAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets/{" + Result.TIMESTAMP + "}")
-                .replaces(
-                    GET,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets/{" + Result.TIMESTAMP + "}",
-                    RestApiVersion.V_7
-                )
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets/{" + Result.TIMESTAMP + "}")
-                .replaces(
-                    POST,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets/{" + Result.TIMESTAMP + "}",
-                    RestApiVersion.V_7
-                )
-                .build(),
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/buckets", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/buckets/{" + TIMESTAMP + "}"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/buckets/{" + TIMESTAMP + "}"),
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/buckets"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/buckets")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java
index 22abcabf5f34f..5eb7f38cd1d1b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.results;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -26,8 +25,8 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.core.ml.action.GetCategoriesAction.Request.CATEGORY_ID;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestGetCategoriesAction extends BaseRestHandler {
@@ -35,26 +34,10 @@ public class RestGetCategoriesAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories/{" + CATEGORY_ID + "}")
-                .replaces(
-                    GET,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories/{" + CATEGORY_ID + "}",
-                    RestApiVersion.V_7
-                )
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories/{" + CATEGORY_ID + "}")
-                .replaces(
-                    POST,
-                    PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories/{" + CATEGORY_ID + "}",
-                    RestApiVersion.V_7
-                )
-                .build(),
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/categories", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/categories/{" + CATEGORY_ID + "}"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/categories/{" + CATEGORY_ID + "}"),
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/categories"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/categories")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java
index 02b57e9c75654..54947d72a96d8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.results;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,8 +22,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestGetInfluencersAction extends BaseRestHandler {
@@ -32,12 +31,8 @@ public class RestGetInfluencersAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/influencers")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/influencers", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/influencers")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/influencers", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/influencers"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/influencers")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java
index c93d7dd57a28d..2700e01cb9f6b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -24,8 +25,8 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -34,12 +35,8 @@ public class RestGetOverallBucketsAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/overall_buckets")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/overall_buckets", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/overall_buckets")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/overall_buckets", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/overall_buckets"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/overall_buckets")
         );
     }
 
@@ -51,6 +48,7 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
         String jobId = restRequest.param(Job.ID.getPreferredName());
+        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
         final Request request;
         if (restRequest.hasContentOrSourceParam()) {
             XContentParser parser = restRequest.contentOrSourceParamParser();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java
index 3b756ddbec17c..84cf13e7032c8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.results;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -23,8 +22,8 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.core.ml.job.config.Job.ID;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestGetRecordsAction extends BaseRestHandler {
@@ -32,12 +31,8 @@ public class RestGetRecordsAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/records")
-                .replaces(GET, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/records", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/records")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/results/records", RestApiVersion.V_7)
-                .build()
+            new Route(GET, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/records"),
+            new Route(POST, BASE_PATH + "anomaly_detectors/{" + ID + "}/results/records")
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateDetectorAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateDetectorAction.java
index 865d630c47816..f1ecf0382cc97 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateDetectorAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateDetectorAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.validate;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,18 +20,13 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestValidateDetectorAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/_validate/detector")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/_validate/detector", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/_validate/detector"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateJobConfigAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateJobConfigAction.java
index 00e776956c7dd..61945ce30faee 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateJobConfigAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateJobConfigAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.ml.rest.validate;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -21,18 +20,13 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH;
-import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestValidateJobConfigAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, BASE_PATH + "anomaly_detectors/_validate")
-                .replaces(POST, PRE_V7_BASE_PATH + "anomaly_detectors/_validate", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, BASE_PATH + "anomaly_detectors/_validate"));
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java
index 32543b45259c2..7e0ff4f029bd4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.ml.MachineLearning;
@@ -103,7 +104,7 @@ protected AbstractJobPersistentTasksExecutor(
         this.expressionResolver = Objects.requireNonNull(expressionResolver);
         this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings);
         this.maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings);
-        this.maxLazyMLNodes = MachineLearning.MAX_LAZY_ML_NODES.get(settings);
+        this.maxLazyMLNodes = MachineLearningField.MAX_LAZY_ML_NODES.get(settings);
         this.maxOpenJobs = MAX_OPEN_JOBS_PER_NODE.get(settings);
         this.useAutoMemoryPercentage = USE_AUTO_MACHINE_MEMORY_PERCENT.get(settings);
         this.maxNodeMemory = MAX_ML_NODE_SIZE.get(settings).getBytes();
@@ -111,7 +112,7 @@ protected AbstractJobPersistentTasksExecutor(
             .addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations);
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(MachineLearning.MAX_MACHINE_MEMORY_PERCENT, this::setMaxMachineMemoryPercent);
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearningField.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_OPEN_JOBS_PER_NODE, this::setMaxOpenJobs);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(USE_AUTO_MACHINE_MEMORY_PERCENT, this::setUseAutoMemoryPercentage);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_ML_NODE_SIZE, this::setMaxNodeSize);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java
index 020f1aae29427..980d7d6b57481 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java
@@ -22,10 +22,10 @@
 
 import java.util.OptionalLong;
 
+import static org.elasticsearch.xpack.core.ml.MachineLearningField.MAX_LAZY_ML_NODES;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT;
 import static org.elasticsearch.xpack.ml.MachineLearning.MACHINE_MEMORY_NODE_ATTR;
 import static org.elasticsearch.xpack.ml.MachineLearning.MAX_JVM_SIZE_NODE_ATTR;
-import static org.elasticsearch.xpack.ml.MachineLearning.MAX_LAZY_ML_NODES;
 import static org.elasticsearch.xpack.ml.MachineLearning.MAX_MACHINE_MEMORY_PERCENT;
 import static org.elasticsearch.xpack.ml.MachineLearning.MAX_ML_NODE_SIZE;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java
index 86488a647baa1..ef6087f021e9d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java
@@ -111,7 +111,7 @@ private SearchResponse initScroll() {
         );
 
         SearchResponse searchResponse = client.search(searchRequest).actionGet();
-        totalHits = searchResponse.getHits().getTotalHits().value;
+        totalHits = searchResponse.getHits().getTotalHits().value();
         scrollId = searchResponse.getScrollId();
         return searchResponse;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java
index f63f6e0549179..802bcaf3b342e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java
@@ -110,7 +110,7 @@ public Deque<T> next() {
         SearchResponse searchResponse = doSearch(searchAfterFields());
         try {
             if (trackTotalHits && totalHits.get() == 0) {
-                totalHits.set(searchResponse.getHits().getTotalHits().value);
+                totalHits.set(searchResponse.getHits().getTotalHits().value());
             }
             return mapHits(searchResponse);
         } finally {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java
index 4a66be4a773f5..2490cd8d5ab21 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsActionTests.java
@@ -148,6 +148,45 @@ public void testAddFailedRoutes_TaskResultIsOverwritten() throws UnknownHostExce
         assertEquals(RoutingState.FAILED, results.get(0).getNodeStats().get(1).getRoutingState().getState());
     }
 
+    public void testAddFailedRoutes_MissingNode() throws UnknownHostException {
+        DiscoveryNodes nodes = buildNodes("node1", "node2");
+        var missingNode = DiscoveryNodeUtils.create(
+            "node3",
+            new TransportAddress(InetAddress.getByAddress(new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 }), 9203)
+        );
+
+        List<AssignmentStats.NodeStats> nodeStatsList = new ArrayList<>();
+        nodeStatsList.add(AssignmentStatsTests.randomNodeStats(nodes.get("node1")));
+        nodeStatsList.add(AssignmentStatsTests.randomNodeStats(nodes.get("node2")));
+
+        var model1 = new AssignmentStats(
+            "model1",
+            "deployment1",
+            randomBoolean() ? null : randomIntBetween(1, 8),
+            randomBoolean() ? null : randomIntBetween(1, 8),
+            null,
+            randomBoolean() ? null : randomIntBetween(1, 10000),
+            randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(1, 1000000)),
+            Instant.now(),
+            nodeStatsList,
+            randomFrom(Priority.values())
+        );
+        var response = new GetDeploymentStatsAction.Response(Collections.emptyList(), Collections.emptyList(), List.of(model1), 1);
+
+        // failed state for node 3 conflicts
+        Map<TrainedModelAssignment, Map<String, RoutingInfo>> badRoutes = new HashMap<>();
+        Map<String, RoutingInfo> nodeRoutes = new HashMap<>();
+        nodeRoutes.put("node3", new RoutingInfo(1, 1, RoutingState.FAILED, "failed on node3"));
+        badRoutes.put(createAssignment("model1"), nodeRoutes);
+
+        var modified = TransportGetDeploymentStatsAction.addFailedRoutes(response, badRoutes, nodes);
+        List<AssignmentStats> results = modified.getStats().results();
+        assertThat(results, hasSize(1));
+        assertThat(results.get(0).getNodeStats(), hasSize(2)); // 3
+        assertEquals("node1", results.get(0).getNodeStats().get(0).getNode().getId());
+        assertEquals("node2", results.get(0).getNodeStats().get(1).getNode().getId());
+    }
+
     private DiscoveryNodes buildNodes(String... nodeIds) throws UnknownHostException {
         InetAddress inetAddress = InetAddress.getByAddress(new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 });
         DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java
index 64d1414134f38..33fae40f80db6 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java
@@ -130,7 +130,7 @@ private static TaskExecutor createTaskExecutor() {
                 MachineLearning.MAX_MACHINE_MEMORY_PERCENT,
                 MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT,
                 MachineLearning.MAX_ML_NODE_SIZE,
-                MachineLearning.MAX_LAZY_ML_NODES,
+                MachineLearningField.MAX_LAZY_ML_NODES,
                 MachineLearning.MAX_OPEN_JOBS_PER_NODE
             )
         );
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java
index 632730bc7f141..cf78e5f900e15 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java
@@ -29,6 +29,12 @@
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderContext;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
+import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
+import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.job.NodeLoad;
@@ -48,6 +54,7 @@
 import static org.elasticsearch.xpack.ml.utils.NativeMemoryCalculator.STATIC_JVM_UPPER_THRESHOLD;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
@@ -262,6 +269,116 @@ public void testScale_GivenUndeterminedMemory_ShouldReturnNullCapacity() {
         assertThat(result.requiredCapacity(), is(nullValue()));
     }
 
+    public void testScale_GivenModelWithZeroAllocations() {
+        MlAutoscalingDeciderService service = buildService();
+        service.onMaster();
+
+        ClusterState clusterState = new ClusterState.Builder(new ClusterName("cluster")).metadata(
+            Metadata.builder()
+                .putCustom(
+                    TrainedModelAssignmentMetadata.NAME,
+                    new TrainedModelAssignmentMetadata(
+                        Map.of(
+                            "model-with-zero-allocations",
+                            TrainedModelAssignment.Builder.empty(
+                                new StartTrainedModelDeploymentAction.TaskParams(
+                                    "model-with-zero-allocations",
+                                    "model-with-zero-allocations-deployment",
+                                    400,
+                                    0,
+                                    2,
+                                    100,
+                                    null,
+                                    Priority.NORMAL,
+                                    0L,
+                                    0L
+                                ),
+                                new AdaptiveAllocationsSettings(true, 0, 4)
+                            ).setAssignmentState(AssignmentState.STARTED).build()
+                        )
+                    )
+                )
+                .build()
+        ).nodes(DiscoveryNodes.builder().add(buildNode("ml-node", ByteSizeValue.ofGb(4), 8)).build()).build();
+
+        AutoscalingDeciderResult result = service.scale(
+            Settings.EMPTY,
+            new DeciderContext(
+                clusterState,
+                new AutoscalingCapacity(
+                    new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null),
+                    new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null)
+                )
+            )
+        );
+        // First call doesn't downscale as delay has not been satisfied
+        assertThat(result.reason().summary(), containsString("down scale delay has not been satisfied"));
+
+        // Let's move time forward 1 hour
+        timeSupplier.setOffset(TimeValue.timeValueHours(1));
+
+        result = service.scale(
+            Settings.EMPTY,
+            new DeciderContext(
+                clusterState,
+                new AutoscalingCapacity(
+                    new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null),
+                    new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null)
+                )
+            )
+        );
+        assertThat(result.reason().summary(), equalTo("Requesting scale down as tier and/or node size could be smaller"));
+        assertThat(result.requiredCapacity().total().memory().getBytes(), equalTo(0L));
+        assertThat(result.requiredCapacity().node().memory().getBytes(), equalTo(0L));
+    }
+
+    public void testScale_GivenTrainedModelAllocationAndNoMlNode() {
+        MlAutoscalingDeciderService service = buildService();
+        service.onMaster();
+
+        ClusterState clusterState = new ClusterState.Builder(new ClusterName("cluster")).metadata(
+            Metadata.builder()
+                .putCustom(
+                    TrainedModelAssignmentMetadata.NAME,
+                    new TrainedModelAssignmentMetadata(
+                        Map.of(
+                            "model",
+                            TrainedModelAssignment.Builder.empty(
+                                new StartTrainedModelDeploymentAction.TaskParams(
+                                    "model",
+                                    "model-deployment",
+                                    400,
+                                    1,
+                                    2,
+                                    100,
+                                    null,
+                                    Priority.NORMAL,
+                                    0L,
+                                    0L
+                                ),
+                                new AdaptiveAllocationsSettings(true, 0, 4)
+                            ).setAssignmentState(AssignmentState.STARTING).build()
+                        )
+                    )
+                )
+                .build()
+        ).build();
+
+        AutoscalingDeciderResult result = service.scale(
+            Settings.EMPTY,
+            new DeciderContext(
+                clusterState,
+                new AutoscalingCapacity(AutoscalingCapacity.AutoscalingResources.ZERO, AutoscalingCapacity.AutoscalingResources.ZERO)
+            )
+        );
+
+        assertThat(result.reason().summary(), containsString("requesting scale up"));
+        assertThat(result.requiredCapacity().total().memory().getBytes(), greaterThan(TEST_JOB_SIZE));
+        assertThat(result.requiredCapacity().total().processors().count(), equalTo(2.0));
+        assertThat(result.requiredCapacity().node().memory().getBytes(), greaterThan(TEST_JOB_SIZE));
+        assertThat(result.requiredCapacity().node().processors().count(), equalTo(2.0));
+    }
+
     private DiscoveryNode buildNode(String id, ByteSizeValue machineMemory, int allocatedProcessors) {
         return DiscoveryNodeUtils.create(
             id,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java
index 3674dda3934bd..729bb708cc46f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.autoscaling.MlAutoscalingStats;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
 import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
 import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo;
@@ -1800,6 +1801,172 @@ public void testGetMemoryAndProcessorsScaleDownNotPreventedByDummyEntityAsMemory
         );
     }
 
+    public void testGetMemoryAndProcessorsScaleDownForModelWithZeroAllocations() throws InterruptedException {
+        long memory = 1000000000;
+        Map<String, String> nodeAttr = Map.of(
+            MachineLearning.MACHINE_MEMORY_NODE_ATTR,
+            Long.toString(memory),
+            MachineLearning.MAX_JVM_SIZE_NODE_ATTR,
+            "400000000",
+            MachineLearning.ML_CONFIG_VERSION_NODE_ATTR,
+            "7.2.0",
+            MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR,
+            "2.0"
+        );
+
+        MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext(
+            List.of(),
+            List.of(),
+            List.of(),
+            Map.of(
+                "model-with-zero-allocations",
+                TrainedModelAssignment.Builder.empty(
+                    new StartTrainedModelDeploymentAction.TaskParams(
+                        "model-with-zero-allocations",
+                        "model-with-zero-allocations-deployment",
+                        400,
+                        0,
+                        2,
+                        100,
+                        null,
+                        Priority.NORMAL,
+                        0L,
+                        0L
+                    ),
+                    new AdaptiveAllocationsSettings(true, 0, 4)
+                ).build()
+            ),
+            List.of(
+                DiscoveryNodeUtils.builder("ml-node-1")
+                    .name("ml-node-name-1")
+                    .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300))
+                    .attributes(nodeAttr)
+                    .roles(Set.of(DiscoveryNodeRole.ML_ROLE))
+                    .build()
+            ),
+            PersistentTasksCustomMetadata.builder().build()
+        );
+        MlMemoryTracker mockTracker = mock(MlMemoryTracker.class);
+
+        this.<MlAutoscalingStats>assertAsync(
+            listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors(
+                mlAutoscalingContext,
+                mockTracker,
+                Map.of("ml-node-1", memory),
+                600000000,
+                2,
+                MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
+                MlDummyAutoscalingEntity.of(0, 0),
+                1,
+                listener
+            ),
+            stats -> {
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(0, stats.currentTotalModelMemoryBytes());
+                assertEquals(0, stats.currentTotalProcessorsInUse());
+                assertEquals(1, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(memory, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
+            }
+        );
+    }
+
+    public void testGetMemoryAndProcessorsIgnoreThreadsOfModelWithZeroAllocations() throws InterruptedException {
+        long memory = 1000000000;
+        Map<String, String> nodeAttr = Map.of(
+            MachineLearning.MACHINE_MEMORY_NODE_ATTR,
+            Long.toString(memory),
+            MachineLearning.MAX_JVM_SIZE_NODE_ATTR,
+            "400000000",
+            MachineLearning.ML_CONFIG_VERSION_NODE_ATTR,
+            "7.2.0",
+            MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR,
+            "2.0"
+        );
+
+        MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext(
+            List.of(),
+            List.of(),
+            List.of(),
+            Map.of(
+                "model-with-one-allocation",
+                TrainedModelAssignment.Builder.empty(
+                    new StartTrainedModelDeploymentAction.TaskParams(
+                        "model-with-one-allocation",
+                        "model-with-one-allocation-deployment",
+                        400,
+                        1,
+                        2,
+                        100,
+                        null,
+                        Priority.NORMAL,
+                        0L,
+                        0L
+                    ),
+                    null
+                ).addRoutingEntry("ml-node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")).build(),
+                "model-with-zero-allocations",
+                TrainedModelAssignment.Builder.empty(
+                    new StartTrainedModelDeploymentAction.TaskParams(
+                        "model-with-zero-allocations",
+                        "model-with-zero-allocations-deployment",
+                        400,
+                        0,
+                        4,
+                        100,
+                        null,
+                        Priority.NORMAL,
+                        0L,
+                        0L
+                    ),
+                    new AdaptiveAllocationsSettings(true, 0, 4)
+                ).build()
+            ),
+            List.of(
+                DiscoveryNodeUtils.builder("ml-node-1")
+                    .name("ml-node-name-1")
+                    .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300))
+                    .attributes(nodeAttr)
+                    .roles(Set.of(DiscoveryNodeRole.ML_ROLE))
+                    .build()
+            ),
+            PersistentTasksCustomMetadata.builder().build()
+        );
+        MlMemoryTracker mockTracker = mock(MlMemoryTracker.class);
+
+        this.<MlAutoscalingStats>assertAsync(
+            listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors(
+                mlAutoscalingContext,
+                mockTracker,
+                Map.of("ml-node-1", memory),
+                600000000,
+                2,
+                MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
+                MlDummyAutoscalingEntity.of(0, 0),
+                1,
+                listener
+            ),
+            stats -> {
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(251659040, stats.currentTotalModelMemoryBytes());
+                assertEquals(2, stats.currentTotalProcessorsInUse());
+                assertEquals(1, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(0, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
+            }
+        );
+    }
+
     private <T> void assertAsync(Consumer<ActionListener<T>> function, Consumer<T> furtherTests) throws InterruptedException {
         CountDownLatch latch = new CountDownLatch(1);
         AtomicBoolean listenerCalled = new AtomicBoolean(false);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java
index 4aaddc91231f3..79f2a913902df 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
@@ -38,6 +40,9 @@
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -66,6 +71,8 @@ public void setUp() throws Exception {
             new ScalingExecutorBuilder(MachineLearning.UTILITY_THREAD_POOL_NAME, 0, 1, TimeValue.timeValueMinutes(10), false)
         );
         clusterService = mock(ClusterService.class);
+        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
+        when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, Set.of()));
         client = mock(Client.class);
         inferenceAuditor = mock(InferenceAuditor.class);
         meterRegistry = mock(MeterRegistry.class);
@@ -240,4 +247,99 @@ public void test() throws IOException {
 
         service.stop();
     }
+
+    public void testMaybeStartAllocation() {
+        AdaptiveAllocationsScalerService service = new AdaptiveAllocationsScalerService(
+            threadPool,
+            clusterService,
+            client,
+            inferenceAuditor,
+            meterRegistry,
+            true,
+            1
+        );
+
+        when(client.threadPool()).thenReturn(threadPool);
+
+        // will not start when adaptive allocations are not enabled
+        assertFalse(service.maybeStartAllocation(TrainedModelAssignment.Builder.empty(taskParams(1), null).build()));
+        assertFalse(
+            service.maybeStartAllocation(
+                TrainedModelAssignment.Builder.empty(taskParams(1), new AdaptiveAllocationsSettings(Boolean.FALSE, 1, 2)).build()
+            )
+        );
+        // min allocations > 0
+        assertFalse(
+            service.maybeStartAllocation(
+                TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 1, 2)).build()
+            )
+        );
+        assertTrue(
+            service.maybeStartAllocation(
+                TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 2)).build()
+            )
+        );
+    }
+
+    public void testMaybeStartAllocation_BlocksMultipleRequests() throws Exception {
+        AdaptiveAllocationsScalerService service = new AdaptiveAllocationsScalerService(
+            threadPool,
+            clusterService,
+            client,
+            inferenceAuditor,
+            meterRegistry,
+            true,
+            1
+        );
+
+        var latch = new CountDownLatch(1);
+        var scalingUpRequestSent = new AtomicBoolean();
+
+        when(client.threadPool()).thenReturn(threadPool);
+        doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            var listener = (ActionListener<CreateTrainedModelAssignmentAction.Response>) invocationOnMock.getArguments()[2];
+            scalingUpRequestSent.set(true);
+            latch.await();
+            listener.onResponse(mock(CreateTrainedModelAssignmentAction.Response.class));
+            return Void.TYPE;
+        }).when(client).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), any(), any());
+
+        threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> {
+            var starting = service.maybeStartAllocation(
+                TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 2)).build()
+            );
+            assertTrue(starting);
+        });
+
+        // wait for the request to be sent
+        assertBusy(() -> assertTrue(scalingUpRequestSent.get()));
+
+        // Due to the inflight request this will not trigger an update request
+        assertTrue(
+            service.maybeStartAllocation(
+                TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 2)).build()
+            )
+        );
+        // release the inflight request
+        latch.countDown();
+
+        verify(client, times(1)).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), any(), any());
+    }
+
+    private StartTrainedModelDeploymentAction.TaskParams taskParams(int numAllocations) {
+        return new StartTrainedModelDeploymentAction.TaskParams(
+            "foo",
+            "foo",
+            1000L,
+            numAllocations,
+            1,
+            100,
+            null,
+            Priority.NORMAL,
+            100L,
+            100L
+        );
+    }
+
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java
index 1887ebe8050e0..a5276e7371128 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java
@@ -7,7 +7,10 @@
 
 package org.elasticsearch.xpack.ml.inference.adaptiveallocations;
 
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
 import java.util.Random;
 
@@ -15,11 +18,21 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class AdaptiveAllocationsScalerTests extends ESTestCase {
 
+    private ClusterService clusterService;
+
+    @Before
+    public void createMocks() throws Exception {
+        clusterService = mock(ClusterService.class);
+        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
+    }
+
     public void testAutoscaling_scaleUpAndDown() {
-        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1);
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60);
 
         // With 1 allocation the system can handle 500 requests * 0.020 sec/request.
         // To handle remaining requests the system should scale to 2 allocations.
@@ -47,7 +60,7 @@ public void testAutoscaling_scaleUpAndDown() {
     }
 
     public void testAutoscaling_noOscillating() {
-        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1);
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60);
 
         // With 1 allocation the system can handle 880 requests * 0.010 sec/request.
         adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(880, 0, 0, 0.010), 10, 1);
@@ -75,7 +88,7 @@ public void testAutoscaling_noOscillating() {
     }
 
     public void testAutoscaling_respectMinMaxAllocations() {
-        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1);
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60);
         adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(2, 5);
 
         // Even though there are no requests, scale to the minimum of 2 allocations.
@@ -98,7 +111,7 @@ public void testAutoscaling_respectMinMaxAllocations() {
     }
 
     public void testEstimation_highVariance() {
-        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1);
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60);
 
         Random random = new Random(42);
 
@@ -140,7 +153,7 @@ public void testEstimation_highVariance() {
     }
 
     public void testAutoscaling_maxAllocationsSafeguard() {
-        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1);
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60);
         adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1_000_000, 10_000_000, 1, 0.05), 10, 1);
         assertThat(adaptiveAllocationsScaler.scale(), equalTo(32));
         adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(2, 77);
@@ -148,7 +161,12 @@ public void testAutoscaling_maxAllocationsSafeguard() {
     }
 
     public void testAutoscaling_scaleDownToZeroAllocations() {
-        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1);
+        int scaleDownAfterInactivitySeconds = 60 * 15; // scale down to 0 after 15 minutes
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler(
+            "test-deployment",
+            1,
+            scaleDownAfterInactivitySeconds
+        );
         // 1 hour with 1 request per 1 seconds, so don't scale.
         for (int i = 0; i < 3600; i++) {
             adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1);
@@ -177,8 +195,37 @@ public void testAutoscaling_scaleDownToZeroAllocations() {
         }
     }
 
+    public void testAutoscaling_resetTimeWithoutRequests() {
+        int scaleDownAfterInactivitySeconds = 60 * 15; // scale down to 0 after 15 minutes
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler(
+            "test-deployment",
+            0,
+            scaleDownAfterInactivitySeconds
+        );
+
+        // 1 hour without requests, but call "reset" every 10 minutes, so don't scale.
+        for (int i = 0; i < 360; i++) {
+            adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 10, 0);
+            assertThat(adaptiveAllocationsScaler.scale(), nullValue());
+            if (i % 60 == 0) {
+                adaptiveAllocationsScaler.resetTimeWithoutRequests();
+            }
+        }
+
+        adaptiveAllocationsScaler.resetTimeWithoutRequests();
+        // 15 minutes with no requests, so don't scale.
+        for (int i = 0; i < 90; i++) {
+            adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 10, 1);
+            assertThat(adaptiveAllocationsScaler.scale(), nullValue());
+        }
+
+        // another second with no requests, so scale to zero allocations.
+        adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 1);
+        assertThat(adaptiveAllocationsScaler.scale(), equalTo(0));
+    }
+
     public void testAutoscaling_dontScaleDownToZeroAllocationsWhenMinAllocationsIsSet() {
-        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1);
+        AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60);
         adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(1, null);
 
         // 1 hour with no requests,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java
index 1dc44582492aa..5a358ef833cb5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java
@@ -52,6 +52,7 @@
 import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelAssignmentRoutingInfoAction;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
 import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
 import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo;
@@ -127,7 +128,7 @@ public void setupObjects() throws IllegalAccessException {
                 MachineLearning.MAX_MACHINE_MEMORY_PERCENT,
                 MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT,
                 MachineLearning.MAX_OPEN_JOBS_PER_NODE,
-                MachineLearning.MAX_LAZY_ML_NODES,
+                MachineLearningField.MAX_LAZY_ML_NODES,
                 MachineLearning.MAX_ML_NODE_SIZE,
                 MachineLearning.ALLOCATED_PROCESSORS_SCALE
             )
@@ -2077,9 +2078,23 @@ private void assertThatStoppingAssignmentPreventsMutation(
         );
     }
 
+    public void testHasUpdates() {
+        var assignment = TrainedModelAssignment.Builder.empty(newParams("foo", 10_000L, 1, 1), null).build();
+        assertFalse(TrainedModelAssignmentClusterService.hasUpdates(1, null, assignment));
+        assertTrue(TrainedModelAssignmentClusterService.hasUpdates(2, null, assignment));
+
+        var adaptiveAllocations = new AdaptiveAllocationsSettings(true, 1, 4);
+        assignment = TrainedModelAssignment.Builder.empty(newParams("foo", 10_000L, 1, 1), adaptiveAllocations).build();
+        assertFalse(TrainedModelAssignmentClusterService.hasUpdates(null, new AdaptiveAllocationsSettings(true, 1, 4), assignment));
+        assertTrue(TrainedModelAssignmentClusterService.hasUpdates(null, new AdaptiveAllocationsSettings(true, 0, 4), assignment));
+
+        assertFalse(TrainedModelAssignmentClusterService.hasUpdates(1, new AdaptiveAllocationsSettings(true, 1, 4), assignment));
+        assertTrue(TrainedModelAssignmentClusterService.hasUpdates(1, new AdaptiveAllocationsSettings(true, 0, 4), assignment));
+    }
+
     private TrainedModelAssignmentClusterService createClusterService(int maxLazyNodes) {
         return new TrainedModelAssignmentClusterService(
-            Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), maxLazyNodes).build(),
+            Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), maxLazyNodes).build(),
             clusterService,
             threadPool,
             nodeLoadDetector,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java
index eb2e21d5fda6c..64251c05af7c8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java
@@ -105,7 +105,7 @@ public void setUpMocks() {
                     ClusterApplierService.CLUSTER_SERVICE_SLOW_TASK_THREAD_DUMP_TIMEOUT_SETTING,
                     MachineLearning.CONCURRENT_JOB_ALLOCATIONS,
                     MachineLearning.MAX_MACHINE_MEMORY_PERCENT,
-                    MachineLearning.MAX_LAZY_ML_NODES,
+                    MachineLearningField.MAX_LAZY_ML_NODES,
                     MachineLearning.MAX_ML_NODE_SIZE,
                     MachineLearning.MAX_OPEN_JOBS_PER_NODE,
                     MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT
@@ -155,7 +155,7 @@ public void testValidate_givenValidJob() {
 
     // An index being unavailable should take precedence over waiting for a lazy node
     public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() {
-        Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 1).build();
+        Settings settings = Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), 1).build();
 
         ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
         Metadata.Builder metadata = Metadata.builder();
@@ -177,7 +177,7 @@ public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() {
     }
 
     public void testGetAssignment_GivenLazyJobAndNoGlobalLazyNodes() {
-        Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 0).build();
+        Settings settings = Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), 0).build();
         ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
         Metadata.Builder metadata = Metadata.builder();
         RoutingTable.Builder routingTable = RoutingTable.builder();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilderTests.java
index 3d17d8dd23ff6..13cf6d87728a8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/SparseVectorQueryBuilderTests.java
@@ -166,8 +166,8 @@ protected void doAssertLuceneQuery(SparseVectorQueryBuilder queryBuilder, Query
         Class<?> boostQueryClass = FeatureField.newLinearQuery("", "", 1.0f).getClass();
 
         for (var clause : booleanQuery.clauses()) {
-            assertEquals(BooleanClause.Occur.SHOULD, clause.getOccur());
-            assertThat(clause.getQuery(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass)));
+            assertEquals(BooleanClause.Occur.SHOULD, clause.occur());
+            assertThat(clause.query(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass)));
         }
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java
index 8da6fc843614e..00d50e0d0d7bb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java
@@ -139,8 +139,8 @@ protected void doAssertLuceneQuery(TextExpansionQueryBuilder queryBuilder, Query
         Class<?> boostQueryClass = FeatureField.newLinearQuery("", "", 1.0f).getClass();
 
         for (var clause : booleanQuery.clauses()) {
-            assertEquals(BooleanClause.Occur.SHOULD, clause.getOccur());
-            assertThat(clause.getQuery(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass)));
+            assertEquals(BooleanClause.Occur.SHOULD, clause.occur());
+            assertThat(clause.query(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass)));
         }
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java
index 7f7c22594abb8..fdb4458a6ec3f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java
@@ -36,11 +36,11 @@
 import java.util.Set;
 import java.util.function.BiConsumer;
 
+import static org.elasticsearch.xpack.core.ml.MachineLearningField.MAX_LAZY_ML_NODES;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.MAX_MODEL_MEMORY_LIMIT;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT;
 import static org.elasticsearch.xpack.ml.MachineLearning.MACHINE_MEMORY_NODE_ATTR;
 import static org.elasticsearch.xpack.ml.MachineLearning.MAX_JVM_SIZE_NODE_ATTR;
-import static org.elasticsearch.xpack.ml.MachineLearning.MAX_LAZY_ML_NODES;
 import static org.elasticsearch.xpack.ml.MachineLearning.MAX_MACHINE_MEMORY_PERCENT;
 import static org.elasticsearch.xpack.ml.MachineLearning.MAX_ML_NODE_SIZE;
 import static org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingDeciderServiceTests.AUTO_NODE_TIERS_NO_MONITORING;
diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
index daea70abd29e3..7ddaa53a59914 100644
--- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
+++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
@@ -149,7 +149,7 @@ public void testMonitoringBulk() throws Exception {
 
                 assertResponse(client().prepareSearch(".monitoring-" + system.getSystem() + "-" + TEMPLATE_VERSION + "-*"), response -> {
                     // exactly 3 results are expected
-                    assertThat("No monitoring documents yet", response.getHits().getTotalHits().value, equalTo(3L));
+                    assertThat("No monitoring documents yet", response.getHits().getTotalHits().value(), equalTo(3L));
 
                     final List<Map<String, Object>> sources = Arrays.stream(response.getHits().getHits())
                         .map(SearchHit::getSourceAsMap)
@@ -165,7 +165,7 @@ public void testMonitoringBulk() throws Exception {
             assertCheckedResponse(client().prepareSearch(monitoringIndex), response -> {
                 final SearchHits hits = response.getHits();
 
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(
                     "Monitoring documents must have the same timestamp",
                     Arrays.stream(hits.getHits()).map(hit -> extractValue("timestamp", hit.getSourceAsMap())).distinct().count(),
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
index 300ac5a650a8d..b69b958a27ce6 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -53,10 +52,7 @@ public class RestMonitoringBulkAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_monitoring/bulk").replaces(POST, "/_xpack/monitoring/_bulk", RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_monitoring/bulk").replaces(PUT, "/_xpack/monitoring/_bulk", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_monitoring/bulk"), new Route(PUT, "/_monitoring/bulk"));
     }
 
     @Override
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
index 93e055b58ddc3..d68395ef7656f 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
@@ -113,7 +113,7 @@ public void testExport() throws Exception {
 
                     assertResponse(
                         prepareSearch(".monitoring-*"),
-                        response -> assertThat((long) nbDocs, lessThanOrEqualTo(response.getHits().getTotalHits().value))
+                        response -> assertThat((long) nbDocs, lessThanOrEqualTo(response.getHits().getTotalHits().value()))
                     );
                 });
 
@@ -260,7 +260,7 @@ private void checkMonitoringDocs() {
         DateFormatter dateFormatter = DateFormatter.forPattern(customTimeFormat).withZone(ZoneOffset.UTC);
 
         assertResponse(prepareSearch(".monitoring-*").setSize(100), rsp -> {
-            assertThat(rsp.getHits().getTotalHits().value, greaterThan(0L));
+            assertThat(rsp.getHits().getTotalHits().value(), greaterThan(0L));
             for (SearchHit hit : rsp.getHits().getHits()) {
                 final Map<String, Object> source = hit.getSourceAsMap();
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
index d6e15ea25c8e1..d382905c1c9c2 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
@@ -293,12 +293,14 @@ private void assertNoWatchesExist() {
             .query(QueryBuilders.matchQuery("metadata.xpack.cluster_uuid", clusterUUID));
 
         assertResponse(prepareSearch(".watches").setSource(searchSource), response -> {
-            if (response.getHits().getTotalHits().value > 0) {
+            if (response.getHits().getTotalHits().value() > 0) {
                 List<String> invalidWatches = new ArrayList<>();
                 for (SearchHit hit : response.getHits().getHits()) {
                     invalidWatches.add(ObjectPath.eval("metadata.xpack.watch", hit.getSourceAsMap()));
                 }
-                fail("Found [" + response.getHits().getTotalHits().value + "] invalid watches when none were expected: " + invalidWatches);
+                fail(
+                    "Found [" + response.getHits().getTotalHits().value() + "] invalid watches when none were expected: " + invalidWatches
+                );
             }
         });
     }
diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java
index 803c7f410c41d..71f788727aa23 100644
--- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java
+++ b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java
@@ -97,7 +97,7 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna
                                     .getAsVersionId(
                                         "version",
                                         IndexVersion::fromId,
-                                        IndexVersion.fromId(randomBoolean() ? 5000099 : 6000099)
+                                        IndexVersion.fromId(randomFrom(5000099, 6000099, 7000099))
                                     )
                             )
                     )
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java
index d1455eaa2f1c4..18adebb145f98 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java
@@ -196,7 +196,7 @@ static final OldSegmentInfos readCommit(Directory directory, String segmentFileN
 
         long generation = generationFromSegmentsFileName(segmentFileName);
         // System.out.println(Thread.currentThread() + ": SegmentInfos.readCommit " + segmentFileName);
-        try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READONCE)) {
+        try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName)) {
             try {
                 return readCommit(directory, input, generation, minSupportedMajorVersion);
             } catch (EOFException | NoSuchFileException | FileNotFoundException e) {
@@ -305,7 +305,7 @@ private static void parseSegmentInfos(Directory directory, DataInput input, OldS
             byte[] segmentID = new byte[StringHelper.ID_LENGTH];
             input.readBytes(segmentID, 0, segmentID.length);
             Codec codec = readCodec(input);
-            SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READ);
+            SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.DEFAULT);
             info.setCodec(codec);
             totalDocs += info.maxDoc();
             long delGen = CodecUtil.readBELong(input);
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java
index 25b4b685ac50f..3ed8fc26ac937 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.lucene.bwc.codecs;
 
-import org.apache.lucene.backward_codecs.lucene70.Lucene70Codec;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FieldsConsumer;
@@ -27,7 +26,6 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -101,6 +99,7 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) {
                     false,
                     fieldInfo.getIndexOptions(),
                     fieldInfo.getDocValuesType(),
+                    fieldInfo.docValuesSkipIndexType(),
                     fieldInfo.getDocValuesGen(),
                     fieldInfo.attributes(),
                     fieldInfo.getPointDimensionCount(),
@@ -119,9 +118,7 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) {
     }
 
     public static SegmentInfo wrap(SegmentInfo segmentInfo) {
-        // special handling for Lucene70Codec (which is currently bundled with Lucene)
-        // Use BWCLucene70Codec instead as that one extends BWCCodec (similar to all other older codecs)
-        final Codec codec = segmentInfo.getCodec() instanceof Lucene70Codec ? new BWCLucene70Codec() : segmentInfo.getCodec();
+        final Codec codec = segmentInfo.getCodec();
         final SegmentInfo segmentInfo1 = new SegmentInfo(
             segmentInfo.dir,
             // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation)
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java
index 5a9b1bb252308..c7abed7d69a59 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyDocValuesIterables.java
@@ -182,10 +182,7 @@ public Number next() {
                         try {
                             if (nextDocID > values.docID()) {
                                 if (values.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
-                                    ordCount = 0;
-                                    while (values.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
-                                        ordCount++;
-                                    }
+                                    ordCount = values.docValueCount();
                                 }
                             }
                             int result;
@@ -225,6 +222,7 @@ public Iterator<Number> iterator() {
 
                 return new Iterator<Number>() {
                     private boolean nextIsSet;
+                    private int currentIndex = 0;
                     private long nextOrd;
 
                     private void setNext() {
@@ -232,17 +230,22 @@ private void setNext() {
                             if (nextIsSet == false) {
                                 if (values.docID() == -1) {
                                     values.nextDoc();
+                                    currentIndex = 0;
                                 }
                                 while (true) {
                                     if (values.docID() == DocIdSetIterator.NO_MORE_DOCS) {
                                         nextOrd = -1;
                                         break;
                                     }
-                                    nextOrd = values.nextOrd();
-                                    if (nextOrd != -1) {
-                                        break;
+                                    if (currentIndex < values.docValueCount()) {
+                                        nextOrd = values.nextOrd();
+                                        currentIndex++;
+                                        if (nextOrd != -1) {
+                                            break;
+                                        }
                                     }
                                     values.nextDoc();
+                                    currentIndex = 0;
                                 }
                                 nextIsSet = true;
                             }
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java
index 21b6818bd5613..80236f3847e12 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java
@@ -53,7 +53,7 @@ public int nextDoc() {
         while (docID < maxDoc) {
             values.setDocument(docID);
             ord = values.nextOrd();
-            if (ord != NO_MORE_ORDS) {
+            if (ord != LegacySortedSetDocValues.NO_MORE_ORDS) {
                 return docID;
             }
             docID++;
@@ -81,7 +81,7 @@ public boolean advanceExact(int target) throws IOException {
         docID = target;
         values.setDocument(docID);
         ord = values.nextOrd();
-        return ord != NO_MORE_ORDS;
+        return ord != LegacySortedSetDocValues.NO_MORE_ORDS;
     }
 
     @Override
@@ -92,7 +92,7 @@ public long cost() {
     @Override
     public long nextOrd() {
         long result = ord;
-        if (result != NO_MORE_ORDS) {
+        if (result != LegacySortedSetDocValues.NO_MORE_ORDS) {
             ord = values.nextOrd();
         }
         return result;
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/ForUtil.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/ForUtil.java
index a567f25869407..007b398624d56 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/ForUtil.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/ForUtil.java
@@ -105,14 +105,14 @@ private static int encodedSize(PackedInts.Format format, int packedIntsVersion,
 
         for (int bpv = 1; bpv <= 32; ++bpv) {
             final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits(BLOCK_SIZE, bpv, acceptableOverheadRatio);
-            assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue);
-            assert formatAndBits.bitsPerValue <= 32;
-            encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue);
-            encoders[bpv] = PackedInts.getEncoder(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue);
-            decoders[bpv] = PackedInts.getDecoder(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue);
+            assert formatAndBits.format().isSupported(formatAndBits.bitsPerValue());
+            assert formatAndBits.bitsPerValue() <= 32;
+            encodedSizes[bpv] = encodedSize(formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());
+            encoders[bpv] = PackedInts.getEncoder(formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());
+            decoders[bpv] = PackedInts.getDecoder(formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());
             iterations[bpv] = computeIterations(decoders[bpv]);
 
-            out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1));
+            out.writeVInt(formatAndBits.format().getId() << 5 | (formatAndBits.bitsPerValue() - 1));
         }
     }
 
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java
index 83fcb17449100..06002d2d10dee 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValuesSkipIndexType;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -103,6 +104,7 @@ public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segm
                             storePayloads,
                             indexOptions,
                             docValuesType,
+                            DocValuesSkipIndexType.NONE,
                             dvGen,
                             attributes,
                             0,
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java
index 09147e821d9fb..607d9903abc87 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java
@@ -28,6 +28,7 @@
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.ImpactsEnum;
@@ -1316,6 +1317,11 @@ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
         }
     }
 
+    @Override
+    public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
+        return null;
+    }
+
     private SortedSetDocValues getSortedSetWithAddresses(FieldInfo field) throws IOException {
         final long valueCount = binaries.get(field.name).count;
         // we keep the byte[]s and list of ords on disk, these could be large
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java
index f3ce3ea0755e1..43203caf571f1 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java
@@ -63,14 +63,14 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException {
         numLeaves = metaIn.readVInt();
         assert numLeaves > 0;
 
-        minPackedValue = new byte[config.packedIndexBytesLength];
-        maxPackedValue = new byte[config.packedIndexBytesLength];
-
-        metaIn.readBytes(minPackedValue, 0, config.packedIndexBytesLength);
-        metaIn.readBytes(maxPackedValue, 0, config.packedIndexBytesLength);
-        final ArrayUtil.ByteArrayComparator comparator = ArrayUtil.getUnsignedComparator(config.bytesPerDim);
-        for (int dim = 0; dim < config.numIndexDims; dim++) {
-            if (comparator.compare(minPackedValue, dim * config.bytesPerDim, maxPackedValue, dim * config.bytesPerDim) > 0) {
+        minPackedValue = new byte[config.packedIndexBytesLength()];
+        maxPackedValue = new byte[config.packedIndexBytesLength()];
+
+        metaIn.readBytes(minPackedValue, 0, config.packedIndexBytesLength());
+        metaIn.readBytes(maxPackedValue, 0, config.packedIndexBytesLength());
+        final ArrayUtil.ByteArrayComparator comparator = ArrayUtil.getUnsignedComparator(config.bytesPerDim());
+        for (int dim = 0; dim < config.numIndexDims(); dim++) {
+            if (comparator.compare(minPackedValue, dim * config.bytesPerDim(), maxPackedValue, dim * config.bytesPerDim()) > 0) {
                 throw new CorruptIndexException(
                     "minPackedValue "
                         + new BytesRef(minPackedValue)
@@ -104,17 +104,17 @@ public byte[] getMaxPackedValue() {
 
     @Override
     public int getNumDimensions() {
-        return config.numDims;
+        return config.numDims();
     }
 
     @Override
     public int getNumIndexDimensions() {
-        return config.numIndexDims;
+        return config.numIndexDims();
     }
 
     @Override
     public int getBytesPerDimension() {
-        return config.bytesPerDim;
+        return config.bytesPerDim();
     }
 
     @Override
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java
index 0e689138acd8f..0100a8bd14635 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java
@@ -32,7 +32,7 @@ public class BWCLucene70Codec extends BWCCodec {
     private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
     private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
     private final StoredFieldsFormat storedFieldsFormat;
-    private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene70");
+    private final DocValuesFormat defaultDVFormat = new Lucene70DocValuesFormat();
     private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
         @Override
         public DocValuesFormat getDocValuesFormatForField(String field) {
@@ -47,7 +47,11 @@ public PostingsFormat getPostingsFormatForField(String field) {
     };
 
     public BWCLucene70Codec() {
-        super("BWCLucene70Codec");
+        this("BWCLucene70Codec");
+    }
+
+    protected BWCLucene70Codec(String name) {
+        super(name);
         storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);
     }
 
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/IndexedDISI.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/IndexedDISI.java
new file mode 100644
index 0000000000000..75119247cdb13
--- /dev/null
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/IndexedDISI.java
@@ -0,0 +1,327 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.RoaringDocIdSet;
+
+import java.io.DataInput;
+import java.io.IOException;
+
+/**
+ * Disk-based implementation of a {@link DocIdSetIterator} which can return the index of the current
+ * document, i.e. the ordinal of the current document among the list of documents that this iterator
+ * can return. This is useful to implement sparse doc values by only having to encode values for
+ * documents that actually have a value.
+ *
+ * <p>Implementation-wise, this {@link DocIdSetIterator} is inspired of {@link RoaringDocIdSet
+ * roaring bitmaps} and encodes ranges of {@code 65536} documents independently and picks between 3
+ * encodings depending on the density of the range:
+ *
+ * <ul>
+ *   <li>{@code ALL} if the range contains 65536 documents exactly,
+ *   <li>{@code DENSE} if the range contains 4096 documents or more; in that case documents are
+ *       stored in a bit set,
+ *   <li>{@code SPARSE} otherwise, and the lower 16 bits of the doc IDs are stored in a {@link
+ *       DataInput#readShort() short}.
+ * </ul>
+ *
+ * <p>Only ranges that contain at least one value are encoded.
+ *
+ * <p>This implementation uses 6 bytes per document in the worst-case, which happens in the case
+ * that all ranges contain exactly one document.
+ */
+final class IndexedDISI extends DocIdSetIterator {
+
+    static final int MAX_ARRAY_LENGTH = (1 << 12) - 1;
+
+    private static void flush(int block, FixedBitSet buffer, int cardinality, IndexOutput out) throws IOException {
+        assert block >= 0 && block < 65536;
+        out.writeShort((short) block);
+        assert cardinality > 0 && cardinality <= 65536;
+        out.writeShort((short) (cardinality - 1));
+        if (cardinality > MAX_ARRAY_LENGTH) {
+            if (cardinality != 65536) { // all docs are set
+                for (long word : buffer.getBits()) {
+                    out.writeLong(word);
+                }
+            }
+        } else {
+            BitSetIterator it = new BitSetIterator(buffer, cardinality);
+            for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+                out.writeShort((short) doc);
+            }
+        }
+    }
+
+    static void writeBitSet(DocIdSetIterator it, IndexOutput out) throws IOException {
+        int i = 0;
+        final FixedBitSet buffer = new FixedBitSet(1 << 16);
+        int prevBlock = -1;
+        for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+            final int block = doc >>> 16;
+            if (prevBlock != -1 && block != prevBlock) {
+                flush(prevBlock, buffer, i, out);
+                buffer.clear(0, buffer.length());
+                prevBlock = block;
+                i = 0;
+            }
+            buffer.set(doc & 0xFFFF);
+            i++;
+            prevBlock = block;
+        }
+        if (i > 0) {
+            flush(prevBlock, buffer, i, out);
+            buffer.clear(0, buffer.length());
+        }
+        // NO_MORE_DOCS is stored explicitly
+        buffer.set(DocIdSetIterator.NO_MORE_DOCS & 0xFFFF);
+        flush(DocIdSetIterator.NO_MORE_DOCS >>> 16, buffer, 1, out);
+    }
+
+    /** The slice that stores the {@link DocIdSetIterator}. */
+    private final IndexInput slice;
+
+    private final long cost;
+
+    IndexedDISI(IndexInput in, long offset, long length, long cost) throws IOException {
+        this(in.slice("docs", offset, length), cost);
+    }
+
+    // This constructor allows to pass the slice directly in case it helps reuse
+    // see eg. Lucene70 norms producer's merge instance
+    IndexedDISI(IndexInput slice, long cost) throws IOException {
+        this.slice = slice;
+        this.cost = cost;
+    }
+
+    private int block = -1;
+    private long blockEnd;
+    private int nextBlockIndex = -1;
+    Method method;
+
+    private int doc = -1;
+    private int index = -1;
+
+    // SPARSE variables
+    boolean exists;
+
+    // DENSE variables
+    private long word;
+    private int wordIndex = -1;
+    // number of one bits encountered so far, including those of `word`
+    private int numberOfOnes;
+
+    // ALL variables
+    private int gap;
+
+    @Override
+    public int docID() {
+        return doc;
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+        final int targetBlock = target & 0xFFFF0000;
+        if (block < targetBlock) {
+            advanceBlock(targetBlock);
+        }
+        if (block == targetBlock) {
+            if (method.advanceWithinBlock(this, target)) {
+                return doc;
+            }
+            readBlockHeader();
+        }
+        boolean found = method.advanceWithinBlock(this, block);
+        assert found;
+        return doc;
+    }
+
+    public boolean advanceExact(int target) throws IOException {
+        final int targetBlock = target & 0xFFFF0000;
+        if (block < targetBlock) {
+            advanceBlock(targetBlock);
+        }
+        boolean found = block == targetBlock && method.advanceExactWithinBlock(this, target);
+        this.doc = target;
+        return found;
+    }
+
+    private void advanceBlock(int targetBlock) throws IOException {
+        do {
+            slice.seek(blockEnd);
+            readBlockHeader();
+        } while (block < targetBlock);
+    }
+
+    private void readBlockHeader() throws IOException {
+        block = Short.toUnsignedInt(slice.readShort()) << 16;
+        assert block >= 0;
+        final int numValues = 1 + Short.toUnsignedInt(slice.readShort());
+        index = nextBlockIndex;
+        nextBlockIndex = index + numValues;
+        if (numValues <= MAX_ARRAY_LENGTH) {
+            method = Method.SPARSE;
+            blockEnd = slice.getFilePointer() + (numValues << 1);
+        } else if (numValues == 65536) {
+            method = Method.ALL;
+            blockEnd = slice.getFilePointer();
+            gap = block - index - 1;
+        } else {
+            method = Method.DENSE;
+            blockEnd = slice.getFilePointer() + (1 << 13);
+            wordIndex = -1;
+            numberOfOnes = index + 1;
+        }
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+        return advance(doc + 1);
+    }
+
+    public int index() {
+        return index;
+    }
+
+    @Override
+    public long cost() {
+        return cost;
+    }
+
+    enum Method {
+        SPARSE {
+            @Override
+            boolean advanceWithinBlock(IndexedDISI disi, int target) throws IOException {
+                final int targetInBlock = target & 0xFFFF;
+                // TODO: binary search
+                for (; disi.index < disi.nextBlockIndex;) {
+                    int doc = Short.toUnsignedInt(disi.slice.readShort());
+                    disi.index++;
+                    if (doc >= targetInBlock) {
+                        disi.doc = disi.block | doc;
+                        disi.exists = true;
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            @Override
+            boolean advanceExactWithinBlock(IndexedDISI disi, int target) throws IOException {
+                final int targetInBlock = target & 0xFFFF;
+                // TODO: binary search
+                if (target == disi.doc) {
+                    return disi.exists;
+                }
+                for (; disi.index < disi.nextBlockIndex;) {
+                    int doc = Short.toUnsignedInt(disi.slice.readShort());
+                    disi.index++;
+                    if (doc >= targetInBlock) {
+                        if (doc != targetInBlock) {
+                            disi.index--;
+                            disi.slice.seek(disi.slice.getFilePointer() - Short.BYTES);
+                            break;
+                        }
+                        disi.exists = true;
+                        return true;
+                    }
+                }
+                disi.exists = false;
+                return false;
+            }
+        },
+        DENSE {
+            @Override
+            boolean advanceWithinBlock(IndexedDISI disi, int target) throws IOException {
+                final int targetInBlock = target & 0xFFFF;
+                final int targetWordIndex = targetInBlock >>> 6;
+                for (int i = disi.wordIndex + 1; i <= targetWordIndex; ++i) {
+                    disi.word = disi.slice.readLong();
+                    disi.numberOfOnes += Long.bitCount(disi.word);
+                }
+                disi.wordIndex = targetWordIndex;
+
+                long leftBits = disi.word >>> target;
+                if (leftBits != 0L) {
+                    disi.doc = target + Long.numberOfTrailingZeros(leftBits);
+                    disi.index = disi.numberOfOnes - Long.bitCount(leftBits);
+                    return true;
+                }
+
+                while (++disi.wordIndex < 1024) {
+                    disi.word = disi.slice.readLong();
+                    if (disi.word != 0) {
+                        disi.index = disi.numberOfOnes;
+                        disi.numberOfOnes += Long.bitCount(disi.word);
+                        disi.doc = disi.block | (disi.wordIndex << 6) | Long.numberOfTrailingZeros(disi.word);
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            @Override
+            boolean advanceExactWithinBlock(IndexedDISI disi, int target) throws IOException {
+                final int targetInBlock = target & 0xFFFF;
+                final int targetWordIndex = targetInBlock >>> 6;
+                for (int i = disi.wordIndex + 1; i <= targetWordIndex; ++i) {
+                    disi.word = disi.slice.readLong();
+                    disi.numberOfOnes += Long.bitCount(disi.word);
+                }
+                disi.wordIndex = targetWordIndex;
+
+                long leftBits = disi.word >>> target;
+                disi.index = disi.numberOfOnes - Long.bitCount(leftBits);
+                return (leftBits & 1L) != 0;
+            }
+        },
+        ALL {
+            @Override
+            boolean advanceWithinBlock(IndexedDISI disi, int target) throws IOException {
+                disi.doc = target;
+                disi.index = target - disi.gap;
+                return true;
+            }
+
+            @Override
+            boolean advanceExactWithinBlock(IndexedDISI disi, int target) throws IOException {
+                disi.index = target - disi.gap;
+                return true;
+            }
+        };
+
+        /**
+         * Advance to the first doc from the block that is equal to or greater than {@code target}.
+         * Return true if there is such a doc and false otherwise.
+         */
+        abstract boolean advanceWithinBlock(IndexedDISI disi, int target) throws IOException;
+
+        /**
+         * Advance the iterator exactly to the position corresponding to the given {@code target} and
+         * return whether this document exists.
+         */
+        abstract boolean advanceExactWithinBlock(IndexedDISI disi, int target) throws IOException;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/EnclosedAgg.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java
similarity index 53%
rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/EnclosedAgg.java
rename to x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java
index 951a991da376b..77de24b53069d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/EnclosedAgg.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70Codec.java
@@ -4,10 +4,12 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.expression.function.aggregate;
 
-// Agg 'enclosed' by another agg. Used for agg that return multiple embedded aggs (like MatrixStats)
-public interface EnclosedAgg {
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70;
 
-    String innerName();
+public class Lucene70Codec extends BWCLucene70Codec {
+
+    public Lucene70Codec() {
+        super("Lucene70");
+    }
 }
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesConsumer.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesConsumer.java
new file mode 100644
index 0000000000000..1d35a60235d35
--- /dev/null
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesConsumer.java
@@ -0,0 +1,681 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70;
+
+import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter;
+import org.apache.lucene.backward_codecs.packed.LegacyDirectWriter;
+import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.EmptyDocValuesProducer;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.store.ByteBuffersDataOutput;
+import org.apache.lucene.store.ByteBuffersIndexOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.MathUtil;
+import org.apache.lucene.util.StringHelper;
+import org.elasticsearch.core.IOUtils;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
+import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70DocValuesFormat.NUMERIC_BLOCK_SHIFT;
+import static org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70DocValuesFormat.NUMERIC_BLOCK_SIZE;
+
+/** writer for {@link Lucene70DocValuesFormat} */
+final class Lucene70DocValuesConsumer extends DocValuesConsumer {
+
+    IndexOutput data, meta;
+    final int maxDoc;
+
+    /** expert: Creates a new writer */
+    Lucene70DocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension)
+        throws IOException {
+        boolean success = false;
+        try {
+            String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+            data = EndiannessReverserUtil.createOutput(state.directory, dataName, state.context);
+            CodecUtil.writeIndexHeader(
+                data,
+                dataCodec,
+                Lucene70DocValuesFormat.VERSION_CURRENT,
+                state.segmentInfo.getId(),
+                state.segmentSuffix
+            );
+            String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+            meta = EndiannessReverserUtil.createOutput(state.directory, metaName, state.context);
+            CodecUtil.writeIndexHeader(
+                meta,
+                metaCodec,
+                Lucene70DocValuesFormat.VERSION_CURRENT,
+                state.segmentInfo.getId(),
+                state.segmentSuffix
+            );
+            maxDoc = state.segmentInfo.maxDoc();
+            success = true;
+        } finally {
+            if (success == false) {
+                IOUtils.closeWhileHandlingException(this);
+            }
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        boolean success = false;
+        try {
+            if (meta != null) {
+                meta.writeInt(-1); // write EOF marker
+                CodecUtil.writeFooter(meta); // write checksum
+            }
+            if (data != null) {
+                CodecUtil.writeFooter(data); // write checksum
+            }
+            success = true;
+        } finally {
+            if (success) {
+                IOUtils.close(data, meta);
+            } else {
+                IOUtils.closeWhileHandlingException(data, meta);
+            }
+            meta = data = null;
+        }
+    }
+
+    @Override
+    public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        meta.writeInt(field.number);
+        meta.writeByte(Lucene70DocValuesFormat.NUMERIC);
+
+        writeValues(field, new EmptyDocValuesProducer() {
+            @Override
+            public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+                return DocValues.singleton(valuesProducer.getNumeric(field));
+            }
+        });
+    }
+
+    private static class MinMaxTracker {
+        long min, max, numValues, spaceInBits;
+
+        MinMaxTracker() {
+            reset();
+            spaceInBits = 0;
+        }
+
+        private void reset() {
+            min = Long.MAX_VALUE;
+            max = Long.MIN_VALUE;
+            numValues = 0;
+        }
+
+        /** Accumulate a new value. */
+        void update(long v) {
+            min = Math.min(min, v);
+            max = Math.max(max, v);
+            ++numValues;
+        }
+
+        /** Update the required space. */
+        void finish() {
+            if (max > min) {
+                spaceInBits += LegacyDirectWriter.unsignedBitsRequired(max - min) * numValues;
+            }
+        }
+
+        /** Update space usage and get ready for accumulating values for the next block. */
+        void nextBlock() {
+            finish();
+            reset();
+        }
+    }
+
+    private long[] writeValues(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
+        int numDocsWithValue = 0;
+        MinMaxTracker minMax = new MinMaxTracker();
+        MinMaxTracker blockMinMax = new MinMaxTracker();
+        long gcd = 0;
+        Set<Long> uniqueValues = new HashSet<>();
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            for (int i = 0, count = values.docValueCount(); i < count; ++i) {
+                long v = values.nextValue();
+
+                if (gcd != 1) {
+                    if (v < Long.MIN_VALUE / 2 || v > Long.MAX_VALUE / 2) {
+                        // in that case v - minValue might overflow and make the GCD computation return
+                        // wrong results. Since these extreme values are unlikely, we just discard
+                        // GCD computation for them
+                        gcd = 1;
+                    } else if (minMax.numValues != 0) { // minValue needs to be set first
+                        gcd = MathUtil.gcd(gcd, v - minMax.min);
+                    }
+                }
+
+                minMax.update(v);
+                blockMinMax.update(v);
+                if (blockMinMax.numValues == NUMERIC_BLOCK_SIZE) {
+                    blockMinMax.nextBlock();
+                }
+
+                if (uniqueValues != null && uniqueValues.add(v) && uniqueValues.size() > 256) {
+                    uniqueValues = null;
+                }
+            }
+
+            numDocsWithValue++;
+        }
+
+        minMax.finish();
+        blockMinMax.finish();
+
+        final long numValues = minMax.numValues;
+        long min = minMax.min;
+        final long max = minMax.max;
+        assert blockMinMax.spaceInBits <= minMax.spaceInBits;
+
+        if (numDocsWithValue == 0) {
+            meta.writeLong(-2);
+            meta.writeLong(0L);
+        } else if (numDocsWithValue == maxDoc) {
+            meta.writeLong(-1);
+            meta.writeLong(0L);
+        } else {
+            long offset = data.getFilePointer();
+            meta.writeLong(offset);
+            values = valuesProducer.getSortedNumeric(field);
+            IndexedDISI.writeBitSet(values, data);
+            meta.writeLong(data.getFilePointer() - offset);
+        }
+
+        meta.writeLong(numValues);
+        final int numBitsPerValue;
+        boolean doBlocks = false;
+        Map<Long, Integer> encode = null;
+        if (min >= max) {
+            numBitsPerValue = 0;
+            meta.writeInt(-1);
+        } else {
+            if (uniqueValues != null
+                && uniqueValues.size() > 1
+                && LegacyDirectWriter.unsignedBitsRequired(uniqueValues.size() - 1) < LegacyDirectWriter.unsignedBitsRequired(
+                    (max - min) / gcd
+                )) {
+                numBitsPerValue = LegacyDirectWriter.unsignedBitsRequired(uniqueValues.size() - 1);
+                final Long[] sortedUniqueValues = uniqueValues.toArray(new Long[0]);
+                Arrays.sort(sortedUniqueValues);
+                meta.writeInt(sortedUniqueValues.length);
+                for (Long v : sortedUniqueValues) {
+                    meta.writeLong(v);
+                }
+                encode = new HashMap<>();
+                for (int i = 0; i < sortedUniqueValues.length; ++i) {
+                    encode.put(sortedUniqueValues[i], i);
+                }
+                min = 0;
+                gcd = 1;
+            } else {
+                uniqueValues = null;
+                // we do blocks if that appears to save 10+% storage
+                doBlocks = minMax.spaceInBits > 0 && (double) blockMinMax.spaceInBits / minMax.spaceInBits <= 0.9;
+                if (doBlocks) {
+                    numBitsPerValue = 0xFF;
+                    meta.writeInt(-2 - NUMERIC_BLOCK_SHIFT);
+                } else {
+                    numBitsPerValue = LegacyDirectWriter.unsignedBitsRequired((max - min) / gcd);
+                    if (gcd == 1
+                        && min > 0
+                        && LegacyDirectWriter.unsignedBitsRequired(max) == LegacyDirectWriter.unsignedBitsRequired(max - min)) {
+                        min = 0;
+                    }
+                    meta.writeInt(-1);
+                }
+            }
+        }
+
+        meta.writeByte((byte) numBitsPerValue);
+        meta.writeLong(min);
+        meta.writeLong(gcd);
+        long startOffset = data.getFilePointer();
+        meta.writeLong(startOffset);
+        if (doBlocks) {
+            writeValuesMultipleBlocks(valuesProducer.getSortedNumeric(field), gcd);
+        } else if (numBitsPerValue != 0) {
+            writeValuesSingleBlock(valuesProducer.getSortedNumeric(field), numValues, numBitsPerValue, min, gcd, encode);
+        }
+        meta.writeLong(data.getFilePointer() - startOffset);
+
+        return new long[] { numDocsWithValue, numValues };
+    }
+
+    private void writeValuesSingleBlock(
+        SortedNumericDocValues values,
+        long numValues,
+        int numBitsPerValue,
+        long min,
+        long gcd,
+        Map<Long, Integer> encode
+    ) throws IOException {
+        LegacyDirectWriter writer = LegacyDirectWriter.getInstance(data, numValues, numBitsPerValue);
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            for (int i = 0, count = values.docValueCount(); i < count; ++i) {
+                long v = values.nextValue();
+                if (encode == null) {
+                    writer.add((v - min) / gcd);
+                } else {
+                    writer.add(encode.get(v));
+                }
+            }
+        }
+        writer.finish();
+    }
+
+    private void writeValuesMultipleBlocks(SortedNumericDocValues values, long gcd) throws IOException {
+        final long[] buffer = new long[NUMERIC_BLOCK_SIZE];
+        final ByteBuffersDataOutput encodeBuffer = ByteBuffersDataOutput.newResettableInstance();
+        int upTo = 0;
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            for (int i = 0, count = values.docValueCount(); i < count; ++i) {
+                buffer[upTo++] = values.nextValue();
+                if (upTo == NUMERIC_BLOCK_SIZE) {
+                    writeBlock(buffer, NUMERIC_BLOCK_SIZE, gcd, encodeBuffer);
+                    upTo = 0;
+                }
+            }
+        }
+        if (upTo > 0) {
+            writeBlock(buffer, upTo, gcd, encodeBuffer);
+        }
+    }
+
+    private void writeBlock(long[] values, int length, long gcd, ByteBuffersDataOutput buffer) throws IOException {
+        assert length > 0;
+        long min = values[0];
+        long max = values[0];
+        for (int i = 1; i < length; ++i) {
+            final long v = values[i];
+            assert Math.floorMod(values[i] - min, gcd) == 0;
+            min = Math.min(min, v);
+            max = Math.max(max, v);
+        }
+        if (min == max) {
+            data.writeByte((byte) 0);
+            data.writeLong(min);
+        } else {
+            final int bitsPerValue = LegacyDirectWriter.unsignedBitsRequired(max - min);
+            buffer.reset();
+            assert buffer.size() == 0;
+            final LegacyDirectWriter w = LegacyDirectWriter.getInstance(buffer, length, bitsPerValue);
+            for (int i = 0; i < length; ++i) {
+                w.add((values[i] - min) / gcd);
+            }
+            w.finish();
+            data.writeByte((byte) bitsPerValue);
+            data.writeLong(min);
+            data.writeInt(Math.toIntExact(buffer.size()));
+            buffer.copyTo(data);
+        }
+    }
+
+    @Override
+    public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        meta.writeInt(field.number);
+        meta.writeByte(Lucene70DocValuesFormat.BINARY);
+
+        BinaryDocValues values = valuesProducer.getBinary(field);
+        long start = data.getFilePointer();
+        meta.writeLong(start);
+        int numDocsWithField = 0;
+        int minLength = Integer.MAX_VALUE;
+        int maxLength = 0;
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            numDocsWithField++;
+            BytesRef v = values.binaryValue();
+            int length = v.length;
+            data.writeBytes(v.bytes, v.offset, v.length);
+            minLength = Math.min(length, minLength);
+            maxLength = Math.max(length, maxLength);
+        }
+        assert numDocsWithField <= maxDoc;
+        meta.writeLong(data.getFilePointer() - start);
+
+        if (numDocsWithField == 0) {
+            meta.writeLong(-2);
+            meta.writeLong(0L);
+        } else if (numDocsWithField == maxDoc) {
+            meta.writeLong(-1);
+            meta.writeLong(0L);
+        } else {
+            long offset = data.getFilePointer();
+            meta.writeLong(offset);
+            values = valuesProducer.getBinary(field);
+            IndexedDISI.writeBitSet(values, data);
+            meta.writeLong(data.getFilePointer() - offset);
+        }
+
+        meta.writeInt(numDocsWithField);
+        meta.writeInt(minLength);
+        meta.writeInt(maxLength);
+        if (maxLength > minLength) {
+            start = data.getFilePointer();
+            meta.writeLong(start);
+            meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
+
+            final LegacyDirectMonotonicWriter writer = LegacyDirectMonotonicWriter.getInstance(
+                meta,
+                data,
+                numDocsWithField + 1,
+                DIRECT_MONOTONIC_BLOCK_SHIFT
+            );
+            long addr = 0;
+            writer.add(addr);
+            values = valuesProducer.getBinary(field);
+            for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+                addr += values.binaryValue().length;
+                writer.add(addr);
+            }
+            writer.finish();
+            meta.writeLong(data.getFilePointer() - start);
+        }
+    }
+
+    @Override
+    public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        meta.writeInt(field.number);
+        meta.writeByte(Lucene70DocValuesFormat.SORTED);
+        doAddSortedField(field, valuesProducer);
+    }
+
+    private void doAddSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        SortedDocValues values = valuesProducer.getSorted(field);
+        int numDocsWithField = 0;
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            numDocsWithField++;
+        }
+
+        if (numDocsWithField == 0) {
+            meta.writeLong(-2);
+            meta.writeLong(0L);
+        } else if (numDocsWithField == maxDoc) {
+            meta.writeLong(-1);
+            meta.writeLong(0L);
+        } else {
+            long offset = data.getFilePointer();
+            meta.writeLong(offset);
+            values = valuesProducer.getSorted(field);
+            IndexedDISI.writeBitSet(values, data);
+            meta.writeLong(data.getFilePointer() - offset);
+        }
+
+        meta.writeInt(numDocsWithField);
+        if (values.getValueCount() <= 1) {
+            meta.writeByte((byte) 0);
+            meta.writeLong(0L);
+            meta.writeLong(0L);
+        } else {
+            int numberOfBitsPerOrd = LegacyDirectWriter.unsignedBitsRequired(values.getValueCount() - 1);
+            meta.writeByte((byte) numberOfBitsPerOrd);
+            long start = data.getFilePointer();
+            meta.writeLong(start);
+            LegacyDirectWriter writer = LegacyDirectWriter.getInstance(data, numDocsWithField, numberOfBitsPerOrd);
+            values = valuesProducer.getSorted(field);
+            for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+                writer.add(values.ordValue());
+            }
+            writer.finish();
+            meta.writeLong(data.getFilePointer() - start);
+        }
+
+        addTermsDict(DocValues.singleton(valuesProducer.getSorted(field)));
+    }
+
+    private void addTermsDict(SortedSetDocValues values) throws IOException {
+        final long size = values.getValueCount();
+        meta.writeVLong(size);
+        meta.writeInt(Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT);
+
+        ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput();
+        ByteBuffersIndexOutput addressIndexOut = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp");
+        meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
+        long numBlocks = (size + Lucene70DocValuesFormat.TERMS_DICT_BLOCK_MASK) >>> Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT;
+        LegacyDirectMonotonicWriter writer = LegacyDirectMonotonicWriter.getInstance(
+            meta,
+            addressIndexOut,
+            numBlocks,
+            DIRECT_MONOTONIC_BLOCK_SHIFT
+        );
+
+        BytesRefBuilder previous = new BytesRefBuilder();
+        long ord = 0;
+        long start = data.getFilePointer();
+        int maxLength = 0;
+        TermsEnum iterator = values.termsEnum();
+        for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
+            if ((ord & Lucene70DocValuesFormat.TERMS_DICT_BLOCK_MASK) == 0) {
+                writer.add(data.getFilePointer() - start);
+                data.writeVInt(term.length);
+                data.writeBytes(term.bytes, term.offset, term.length);
+            } else {
+                final int prefixLength = StringHelper.bytesDifference(previous.get(), term);
+                final int suffixLength = term.length - prefixLength;
+                assert suffixLength > 0; // terms are unique
+
+                data.writeByte((byte) (Math.min(prefixLength, 15) | (Math.min(15, suffixLength - 1) << 4)));
+                if (prefixLength >= 15) {
+                    data.writeVInt(prefixLength - 15);
+                }
+                if (suffixLength >= 16) {
+                    data.writeVInt(suffixLength - 16);
+                }
+                data.writeBytes(term.bytes, term.offset + prefixLength, term.length - prefixLength);
+            }
+            maxLength = Math.max(maxLength, term.length);
+            previous.copyBytes(term);
+            ++ord;
+        }
+        writer.finish();
+        meta.writeInt(maxLength);
+        meta.writeLong(start);
+        meta.writeLong(data.getFilePointer() - start);
+        start = data.getFilePointer();
+        addressBuffer.copyTo(data);
+        meta.writeLong(start);
+        meta.writeLong(data.getFilePointer() - start);
+
+        // Now write the reverse terms index
+        writeTermsIndex(values);
+    }
+
+    private void writeTermsIndex(SortedSetDocValues values) throws IOException {
+        final long size = values.getValueCount();
+        meta.writeInt(Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
+        long start = data.getFilePointer();
+
+        long numBlocks = 1L + ((size + Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK)
+            >>> Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
+        ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput();
+        ByteBuffersIndexOutput addressIndexOut = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp");
+        LegacyDirectMonotonicWriter writer = LegacyDirectMonotonicWriter.getInstance(
+            meta,
+            addressIndexOut,
+            numBlocks,
+            DIRECT_MONOTONIC_BLOCK_SHIFT
+        );
+
+        TermsEnum iterator = values.termsEnum();
+        BytesRefBuilder previous = new BytesRefBuilder();
+        long offset = 0;
+        long ord = 0;
+        for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
+            if ((ord & Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) {
+                writer.add(offset);
+                final int sortKeyLength;
+                if (ord == 0) {
+                    // no previous term: no bytes to write
+                    sortKeyLength = 0;
+                } else {
+                    sortKeyLength = StringHelper.sortKeyLength(previous.get(), term);
+                }
+                offset += sortKeyLength;
+                data.writeBytes(term.bytes, term.offset, sortKeyLength);
+            } else if ((ord
+                & Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) {
+                    previous.copyBytes(term);
+                }
+            ++ord;
+        }
+        writer.add(offset);
+        writer.finish();
+        meta.writeLong(start);
+        meta.writeLong(data.getFilePointer() - start);
+        start = data.getFilePointer();
+        addressBuffer.copyTo(data);
+        meta.writeLong(start);
+        meta.writeLong(data.getFilePointer() - start);
+    }
+
+    @Override
+    public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        meta.writeInt(field.number);
+        meta.writeByte(Lucene70DocValuesFormat.SORTED_NUMERIC);
+
+        long[] stats = writeValues(field, valuesProducer);
+        int numDocsWithField = Math.toIntExact(stats[0]);
+        long numValues = stats[1];
+        assert numValues >= numDocsWithField;
+
+        meta.writeInt(numDocsWithField);
+        if (numValues > numDocsWithField) {
+            long start = data.getFilePointer();
+            meta.writeLong(start);
+            meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
+
+            final LegacyDirectMonotonicWriter addressesWriter = LegacyDirectMonotonicWriter.getInstance(
+                meta,
+                data,
+                numDocsWithField + 1L,
+                DIRECT_MONOTONIC_BLOCK_SHIFT
+            );
+            long addr = 0;
+            addressesWriter.add(addr);
+            SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
+            for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+                addr += values.docValueCount();
+                addressesWriter.add(addr);
+            }
+            addressesWriter.finish();
+            meta.writeLong(data.getFilePointer() - start);
+        }
+    }
+
+    @Override
+    public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        meta.writeInt(field.number);
+        meta.writeByte(Lucene70DocValuesFormat.SORTED_SET);
+
+        SortedSetDocValues values = valuesProducer.getSortedSet(field);
+        int numDocsWithField = 0;
+        long numOrds = 0;
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            numDocsWithField++;
+            numOrds += values.docValueCount();
+        }
+
+        if (numDocsWithField == numOrds) {
+            meta.writeByte((byte) 0);
+            doAddSortedField(field, new EmptyDocValuesProducer() {
+                @Override
+                public SortedDocValues getSorted(FieldInfo field) throws IOException {
+                    return SortedSetSelector.wrap(valuesProducer.getSortedSet(field), SortedSetSelector.Type.MIN);
+                }
+            });
+            return;
+        }
+        meta.writeByte((byte) 1);
+
+        assert numDocsWithField != 0;
+        if (numDocsWithField == maxDoc) {
+            meta.writeLong(-1);
+            meta.writeLong(0L);
+        } else {
+            long offset = data.getFilePointer();
+            meta.writeLong(offset);
+            values = valuesProducer.getSortedSet(field);
+            IndexedDISI.writeBitSet(values, data);
+            meta.writeLong(data.getFilePointer() - offset);
+        }
+
+        int numberOfBitsPerOrd = LegacyDirectWriter.unsignedBitsRequired(values.getValueCount() - 1);
+        meta.writeByte((byte) numberOfBitsPerOrd);
+        long start = data.getFilePointer();
+        meta.writeLong(start);
+        LegacyDirectWriter writer = LegacyDirectWriter.getInstance(data, numOrds, numberOfBitsPerOrd);
+        values = valuesProducer.getSortedSet(field);
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            for (int i = 0; i < values.docValueCount(); i++) {
+                writer.add(values.nextOrd());
+            }
+        }
+        writer.finish();
+        meta.writeLong(data.getFilePointer() - start);
+
+        meta.writeInt(numDocsWithField);
+        start = data.getFilePointer();
+        meta.writeLong(start);
+        meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
+
+        final LegacyDirectMonotonicWriter addressesWriter = LegacyDirectMonotonicWriter.getInstance(
+            meta,
+            data,
+            numDocsWithField + 1,
+            DIRECT_MONOTONIC_BLOCK_SHIFT
+        );
+        long addr = 0;
+        addressesWriter.add(addr);
+        values = valuesProducer.getSortedSet(field);
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            values.nextOrd();
+            addr += values.docValueCount();
+            addressesWriter.add(addr);
+        }
+        addressesWriter.finish();
+        meta.writeLong(data.getFilePointer() - start);
+
+        addTermsDict(values);
+    }
+}
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesFormat.java
new file mode 100644
index 0000000000000..76fce4cd15c93
--- /dev/null
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesFormat.java
@@ -0,0 +1,171 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70;
+
+import org.apache.lucene.backward_codecs.packed.LegacyDirectWriter;
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.SmallFloat;
+
+import java.io.IOException;
+
+/**
+ * Lucene 7.0 DocValues format.
+ *
+ * <p>Documents that have a value for the field are encoded in a way that it is always possible to
+ * know the ordinal of the current document in the set of documents that have a value. For instance,
+ * say the set of documents that have a value for the field is <code>{1, 5, 6, 11}</code>. When the
+ * iterator is on <code>6</code>, it knows that this is the 3rd item of the set. This way, values
+ * can be stored densely and accessed based on their index at search time. If all documents in a
+ * segment have a value for the field, the index is the same as the doc ID, so this case is encoded
+ * implicitly and is very fast at query time. On the other hand if some documents are missing a
+ * value for the field then the set of documents that have a value is encoded into blocks. All doc
+ * IDs that share the same upper 16 bits are encoded into the same block with the following
+ * strategies:
+ *
+ * <ul>
+ *   <li>SPARSE: This strategy is used when a block contains at most 4095 documents. The lower 16
+ *       bits of doc IDs are stored as {@link DataOutput#writeShort(short) shorts} while the upper
+ *       16 bits are given by the block ID.
+ *   <li>DENSE: This strategy is used when a block contains between 4096 and 65535 documents. The
+ *       lower bits of doc IDs are stored in a bit set. Advancing is performed using {@link
+ *       Long#numberOfTrailingZeros(long) ntz} operations while the index is computed by
+ *       accumulating the {@link Long#bitCount(long) bit counts} of the visited longs.
+ *   <li>ALL: This strategy is used when a block contains exactly 65536 documents, meaning that the
+ *       block is full. In that case doc IDs do not need to be stored explicitly. This is typically
+ *       faster than both SPARSE and DENSE which is a reason why it is preferable to have all
+ *       documents that have a value for a field using contiguous doc IDs, for instance by using
+ *       {@link IndexWriterConfig#setIndexSort(org.apache.lucene.search.Sort) index sorting}.
+ * </ul>
+ *
+ * <p>Then the five per-document value types (Numeric,Binary,Sorted,SortedSet,SortedNumeric) are
+ * encoded using the following strategies:
+ *
+ * <p>{@link DocValuesType#NUMERIC NUMERIC}:
+ *
+ * <ul>
+ *   <li>Delta-compressed: per-document integers written as deltas from the minimum value,
+ *       compressed with bitpacking. For more information, see {@link LegacyDirectWriter}.
+ *   <li>Table-compressed: when the number of unique values is very small (&lt; 256), and when there
+ *       are unused "gaps" in the range of values used (such as {@link SmallFloat}), a lookup table
+ *       is written instead. Each per-document entry is instead the ordinal to this table, and those
+ *       ordinals are compressed with bitpacking ({@link LegacyDirectWriter}).
+ *   <li>GCD-compressed: when all numbers share a common divisor, such as dates, the greatest common
+ *       denominator (GCD) is computed, and quotients are stored using Delta-compressed Numerics.
+ *   <li>Monotonic-compressed: when all numbers are monotonically increasing offsets, they are
+ *       written as blocks of bitpacked integers, encoding the deviation from the expected delta.
+ *   <li>Const-compressed: when there is only one possible value, no per-document data is needed and
+ *       this value is encoded alone.
+ * </ul>
+ *
+ * <p>{@link DocValuesType#BINARY BINARY}:
+ *
+ * <ul>
+ *   <li>Fixed-width Binary: one large concatenated byte[] is written, along with the fixed length.
+ *       Each document's value can be addressed directly with multiplication ({@code docID *
+ *       length}).
+ *   <li>Variable-width Binary: one large concatenated byte[] is written, along with end addresses
+ *       for each document. The addresses are written as Monotonic-compressed numerics.
+ *   <li>Prefix-compressed Binary: values are written in chunks of 16, with the first value written
+ *       completely and other values sharing prefixes. chunk addresses are written as
+ *       Monotonic-compressed numerics. A reverse lookup index is written from a portion of every
+ *       1024th term.
+ * </ul>
+ *
+ * <p>{@link DocValuesType#SORTED SORTED}:
+ *
+ * <ul>
+ *   <li>Sorted: a mapping of ordinals to deduplicated terms is written as Prefix-compressed Binary,
+ *       along with the per-document ordinals written using one of the numeric strategies above.
+ * </ul>
+ *
+ * <p>{@link DocValuesType#SORTED_SET SORTED_SET}:
+ *
+ * <ul>
+ *   <li>Single: if all documents have 0 or 1 value, then data are written like SORTED.
+ *   <li>SortedSet: a mapping of ordinals to deduplicated terms is written as Binary, an ordinal
+ *       list and per-document index into this list are written using the numeric strategies above.
+ * </ul>
+ *
+ * <p>{@link DocValuesType#SORTED_NUMERIC SORTED_NUMERIC}:
+ *
+ * <ul>
+ *   <li>Single: if all documents have 0 or 1 value, then data are written like NUMERIC.
+ *   <li>SortedNumeric: a value list and per-document index into this list are written using the
+ *       numeric strategies above.
+ * </ul>
+ *
+ * <p>Files:
+ *
+ * <ol>
+ *   <li><code>.dvd</code>: DocValues data
+ *   <li><code>.dvm</code>: DocValues metadata
+ * </ol>
+ */
+public final class Lucene70DocValuesFormat extends DocValuesFormat {
+
+    /** Sole Constructor */
+    public Lucene70DocValuesFormat() {
+        super("Lucene70");
+    }
+
+    @Override
+    public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+        return new Lucene70DocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+    }
+
+    @Override
+    public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
+        return new Lucene70DocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+    }
+
+    static final String DATA_CODEC = "Lucene70DocValuesData";
+    static final String DATA_EXTENSION = "dvd";
+    static final String META_CODEC = "Lucene70DocValuesMetadata";
+    static final String META_EXTENSION = "dvm";
+    static final int VERSION_START = 0;
+    static final int VERSION_CURRENT = VERSION_START;
+
+    // indicates docvalues type
+    static final byte NUMERIC = 0;
+    static final byte BINARY = 1;
+    static final byte SORTED = 2;
+    static final byte SORTED_SET = 3;
+    static final byte SORTED_NUMERIC = 4;
+
+    static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
+
+    static final int NUMERIC_BLOCK_SHIFT = 14;
+    static final int NUMERIC_BLOCK_SIZE = 1 << NUMERIC_BLOCK_SHIFT;
+
+    static final int TERMS_DICT_BLOCK_SHIFT = 4;
+    static final int TERMS_DICT_BLOCK_SIZE = 1 << TERMS_DICT_BLOCK_SHIFT;
+    static final int TERMS_DICT_BLOCK_MASK = TERMS_DICT_BLOCK_SIZE - 1;
+
+    static final int TERMS_DICT_REVERSE_INDEX_SHIFT = 10;
+    static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;
+    static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;
+}
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesProducer.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesProducer.java
new file mode 100644
index 0000000000000..5164a67c428b3
--- /dev/null
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesProducer.java
@@ -0,0 +1,1461 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70;
+
+import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;
+import org.apache.lucene.backward_codecs.packed.LegacyDirectReader;
+import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.BaseTermsEnum;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LongValues;
+import org.elasticsearch.core.IOUtils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/** reader for {@link Lucene70DocValuesFormat} */
+final class Lucene70DocValuesProducer extends DocValuesProducer {
+    private final Map<String, NumericEntry> numerics = new HashMap<>();
+    private final Map<String, BinaryEntry> binaries = new HashMap<>();
+    private final Map<String, SortedEntry> sorted = new HashMap<>();
+    private final Map<String, SortedSetEntry> sortedSets = new HashMap<>();
+    private final Map<String, SortedNumericEntry> sortedNumerics = new HashMap<>();
+    private final IndexInput data;
+    private final int maxDoc;
+
+    static final long NO_MORE_ORDS = -1;
+
+    /** expert: instantiates a new reader */
+    Lucene70DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension)
+        throws IOException {
+        String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+        this.maxDoc = state.segmentInfo.maxDoc();
+
+        int version = -1;
+
+        // read in the entries from the metadata file.
+        try (ChecksumIndexInput in = EndiannessReverserUtil.openChecksumInput(state.directory, metaName, state.context)) {
+            Throwable priorE = null;
+            try {
+                version = CodecUtil.checkIndexHeader(
+                    in,
+                    metaCodec,
+                    Lucene70DocValuesFormat.VERSION_START,
+                    Lucene70DocValuesFormat.VERSION_CURRENT,
+                    state.segmentInfo.getId(),
+                    state.segmentSuffix
+                );
+                readFields(in, state.fieldInfos);
+            } catch (Throwable exception) {
+                priorE = exception;
+            } finally {
+                CodecUtil.checkFooter(in, priorE);
+            }
+        }
+
+        String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+        this.data = EndiannessReverserUtil.openInput(state.directory, dataName, state.context);
+        boolean success = false;
+        try {
+            final int version2 = CodecUtil.checkIndexHeader(
+                data,
+                dataCodec,
+                Lucene70DocValuesFormat.VERSION_START,
+                Lucene70DocValuesFormat.VERSION_CURRENT,
+                state.segmentInfo.getId(),
+                state.segmentSuffix
+            );
+            if (version != version2) {
+                throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
+            }
+
+            // NOTE: data file is too costly to verify checksum against all the bytes on open,
+            // but for now we at least verify proper structure of the checksum footer: which looks
+            // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+            // such as file truncation.
+            CodecUtil.retrieveChecksum(data);
+
+            success = true;
+        } finally {
+            if (success == false) {
+                IOUtils.closeWhileHandlingException(this.data);
+            }
+        }
+    }
+
+    private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException {
+        for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
+            FieldInfo info = infos.fieldInfo(fieldNumber);
+            if (info == null) {
+                throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
+            }
+            byte type = meta.readByte();
+            if (type == Lucene70DocValuesFormat.NUMERIC) {
+                numerics.put(info.name, readNumeric(meta));
+            } else if (type == Lucene70DocValuesFormat.BINARY) {
+                binaries.put(info.name, readBinary(meta));
+            } else if (type == Lucene70DocValuesFormat.SORTED) {
+                sorted.put(info.name, readSorted(meta));
+            } else if (type == Lucene70DocValuesFormat.SORTED_SET) {
+                sortedSets.put(info.name, readSortedSet(meta));
+            } else if (type == Lucene70DocValuesFormat.SORTED_NUMERIC) {
+                sortedNumerics.put(info.name, readSortedNumeric(meta));
+            } else {
+                throw new CorruptIndexException("invalid type: " + type, meta);
+            }
+        }
+    }
+
+    private NumericEntry readNumeric(ChecksumIndexInput meta) throws IOException {
+        NumericEntry entry = new NumericEntry();
+        readNumeric(meta, entry);
+        return entry;
+    }
+
+    private void readNumeric(ChecksumIndexInput meta, NumericEntry entry) throws IOException {
+        entry.docsWithFieldOffset = meta.readLong();
+        entry.docsWithFieldLength = meta.readLong();
+        entry.numValues = meta.readLong();
+        int tableSize = meta.readInt();
+        if (tableSize > 256) {
+            throw new CorruptIndexException("invalid table size: " + tableSize, meta);
+        }
+        if (tableSize >= 0) {
+            entry.table = new long[tableSize];
+            for (int i = 0; i < tableSize; ++i) {
+                entry.table[i] = meta.readLong();
+            }
+        }
+        if (tableSize < -1) {
+            entry.blockShift = -2 - tableSize;
+        } else {
+            entry.blockShift = -1;
+        }
+        entry.bitsPerValue = meta.readByte();
+        entry.minValue = meta.readLong();
+        entry.gcd = meta.readLong();
+        entry.valuesOffset = meta.readLong();
+        entry.valuesLength = meta.readLong();
+    }
+
+    private BinaryEntry readBinary(ChecksumIndexInput meta) throws IOException {
+        BinaryEntry entry = new BinaryEntry();
+        entry.dataOffset = meta.readLong();
+        entry.dataLength = meta.readLong();
+        entry.docsWithFieldOffset = meta.readLong();
+        entry.docsWithFieldLength = meta.readLong();
+        entry.numDocsWithField = meta.readInt();
+        entry.minLength = meta.readInt();
+        entry.maxLength = meta.readInt();
+        if (entry.minLength < entry.maxLength) {
+            entry.addressesOffset = meta.readLong();
+            final int blockShift = meta.readVInt();
+            entry.addressesMeta = LegacyDirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1L, blockShift);
+            entry.addressesLength = meta.readLong();
+        }
+        return entry;
+    }
+
+    private SortedEntry readSorted(ChecksumIndexInput meta) throws IOException {
+        SortedEntry entry = new SortedEntry();
+        entry.docsWithFieldOffset = meta.readLong();
+        entry.docsWithFieldLength = meta.readLong();
+        entry.numDocsWithField = meta.readInt();
+        entry.bitsPerValue = meta.readByte();
+        entry.ordsOffset = meta.readLong();
+        entry.ordsLength = meta.readLong();
+        readTermDict(meta, entry);
+        return entry;
+    }
+
+    private SortedSetEntry readSortedSet(ChecksumIndexInput meta) throws IOException {
+        SortedSetEntry entry = new SortedSetEntry();
+        byte multiValued = meta.readByte();
+        switch (multiValued) {
+            case 0: // singlevalued
+                entry.singleValueEntry = readSorted(meta);
+                return entry;
+            case 1: // multivalued
+                break;
+            default:
+                throw new CorruptIndexException("Invalid multiValued flag: " + multiValued, meta);
+        }
+        entry.docsWithFieldOffset = meta.readLong();
+        entry.docsWithFieldLength = meta.readLong();
+        entry.bitsPerValue = meta.readByte();
+        entry.ordsOffset = meta.readLong();
+        entry.ordsLength = meta.readLong();
+        entry.numDocsWithField = meta.readInt();
+        entry.addressesOffset = meta.readLong();
+        final int blockShift = meta.readVInt();
+        entry.addressesMeta = LegacyDirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift);
+        entry.addressesLength = meta.readLong();
+        readTermDict(meta, entry);
+        return entry;
+    }
+
+    private static void readTermDict(ChecksumIndexInput meta, TermsDictEntry entry) throws IOException {
+        entry.termsDictSize = meta.readVLong();
+        entry.termsDictBlockShift = meta.readInt();
+        final int blockShift = meta.readInt();
+        final long addressesSize = (entry.termsDictSize + (1L << entry.termsDictBlockShift) - 1) >>> entry.termsDictBlockShift;
+        entry.termsAddressesMeta = LegacyDirectMonotonicReader.loadMeta(meta, addressesSize, blockShift);
+        entry.maxTermLength = meta.readInt();
+        entry.termsDataOffset = meta.readLong();
+        entry.termsDataLength = meta.readLong();
+        entry.termsAddressesOffset = meta.readLong();
+        entry.termsAddressesLength = meta.readLong();
+        entry.termsDictIndexShift = meta.readInt();
+        final long indexSize = (entry.termsDictSize + (1L << entry.termsDictIndexShift) - 1) >>> entry.termsDictIndexShift;
+        entry.termsIndexAddressesMeta = LegacyDirectMonotonicReader.loadMeta(meta, 1 + indexSize, blockShift);
+        entry.termsIndexOffset = meta.readLong();
+        entry.termsIndexLength = meta.readLong();
+        entry.termsIndexAddressesOffset = meta.readLong();
+        entry.termsIndexAddressesLength = meta.readLong();
+    }
+
+    private SortedNumericEntry readSortedNumeric(ChecksumIndexInput meta) throws IOException {
+        SortedNumericEntry entry = new SortedNumericEntry();
+        readNumeric(meta, entry);
+        entry.numDocsWithField = meta.readInt();
+        if (entry.numDocsWithField != entry.numValues) {
+            entry.addressesOffset = meta.readLong();
+            final int blockShift = meta.readVInt();
+            entry.addressesMeta = LegacyDirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift);
+            entry.addressesLength = meta.readLong();
+        }
+        return entry;
+    }
+
+    @Override
+    public void close() throws IOException {
+        data.close();
+    }
+
+    private static class NumericEntry {
+        long[] table;
+        int blockShift;
+        byte bitsPerValue;
+        long docsWithFieldOffset;
+        long docsWithFieldLength;
+        long numValues;
+        long minValue;
+        long gcd;
+        long valuesOffset;
+        long valuesLength;
+    }
+
+    private static class BinaryEntry {
+        long dataOffset;
+        long dataLength;
+        long docsWithFieldOffset;
+        long docsWithFieldLength;
+        int numDocsWithField;
+        int minLength;
+        int maxLength;
+        long addressesOffset;
+        long addressesLength;
+        LegacyDirectMonotonicReader.Meta addressesMeta;
+    }
+
+    private static class TermsDictEntry {
+        long termsDictSize;
+        int termsDictBlockShift;
+        LegacyDirectMonotonicReader.Meta termsAddressesMeta;
+        int maxTermLength;
+        long termsDataOffset;
+        long termsDataLength;
+        long termsAddressesOffset;
+        long termsAddressesLength;
+        int termsDictIndexShift;
+        LegacyDirectMonotonicReader.Meta termsIndexAddressesMeta;
+        long termsIndexOffset;
+        long termsIndexLength;
+        long termsIndexAddressesOffset;
+        long termsIndexAddressesLength;
+    }
+
+    private static class SortedEntry extends TermsDictEntry {
+        long docsWithFieldOffset;
+        long docsWithFieldLength;
+        int numDocsWithField;
+        byte bitsPerValue;
+        long ordsOffset;
+        long ordsLength;
+    }
+
+    private static class SortedSetEntry extends TermsDictEntry {
+        SortedEntry singleValueEntry;
+        long docsWithFieldOffset;
+        long docsWithFieldLength;
+        int numDocsWithField;
+        byte bitsPerValue;
+        long ordsOffset;
+        long ordsLength;
+        LegacyDirectMonotonicReader.Meta addressesMeta;
+        long addressesOffset;
+        long addressesLength;
+    }
+
+    private static class SortedNumericEntry extends NumericEntry {
+        int numDocsWithField;
+        LegacyDirectMonotonicReader.Meta addressesMeta;
+        long addressesOffset;
+        long addressesLength;
+    }
+
+    @Override
+    public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+        NumericEntry entry = numerics.get(field.name);
+        return getNumeric(entry);
+    }
+
+    private abstract static class DenseNumericDocValues extends NumericDocValues {
+
+        final int maxDoc;
+        int doc = -1;
+
+        DenseNumericDocValues(int maxDoc) {
+            this.maxDoc = maxDoc;
+        }
+
+        @Override
+        public int docID() {
+            return doc;
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            return advance(doc + 1);
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            if (target >= maxDoc) {
+                return doc = NO_MORE_DOCS;
+            }
+            return doc = target;
+        }
+
+        @Override
+        public boolean advanceExact(int target) {
+            doc = target;
+            return true;
+        }
+
+        @Override
+        public long cost() {
+            return maxDoc;
+        }
+    }
+
+    private abstract static class SparseNumericDocValues extends NumericDocValues {
+
+        final IndexedDISI disi;
+
+        SparseNumericDocValues(IndexedDISI disi) {
+            this.disi = disi;
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            return disi.advance(target);
+        }
+
+        @Override
+        public boolean advanceExact(int target) throws IOException {
+            return disi.advanceExact(target);
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            return disi.nextDoc();
+        }
+
+        @Override
+        public int docID() {
+            return disi.docID();
+        }
+
+        @Override
+        public long cost() {
+            return disi.cost();
+        }
+    }
+
+    private NumericDocValues getNumeric(NumericEntry entry) throws IOException {
+        if (entry.docsWithFieldOffset == -2) {
+            // empty
+            return DocValues.emptyNumeric();
+        } else if (entry.docsWithFieldOffset == -1) {
+            // dense
+            if (entry.bitsPerValue == 0) {
+                return new DenseNumericDocValues(maxDoc) {
+                    @Override
+                    public long longValue() throws IOException {
+                        return entry.minValue;
+                    }
+                };
+            } else {
+                final RandomAccessInput slice = data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
+                if (entry.blockShift >= 0) {
+                    // dense but split into blocks of different bits per value
+                    final int shift = entry.blockShift;
+                    final long mul = entry.gcd;
+                    final int mask = (1 << shift) - 1;
+                    return new DenseNumericDocValues(maxDoc) {
+                        int block = -1;
+                        long delta;
+                        long offset;
+                        long blockEndOffset;
+                        LongValues values;
+
+                        @Override
+                        public long longValue() throws IOException {
+                            final int block = doc >>> shift;
+                            if (this.block != block) {
+                                int bitsPerValue;
+                                do {
+                                    offset = blockEndOffset;
+                                    bitsPerValue = slice.readByte(offset++);
+                                    delta = slice.readLong(offset);
+                                    offset += Long.BYTES;
+                                    if (bitsPerValue == 0) {
+                                        blockEndOffset = offset;
+                                    } else {
+                                        final int length = slice.readInt(offset);
+                                        offset += Integer.BYTES;
+                                        blockEndOffset = offset + length;
+                                    }
+                                    this.block++;
+                                } while (this.block != block);
+                                values = bitsPerValue == 0
+                                    ? LongValues.ZEROES
+                                    : LegacyDirectReader.getInstance(slice, bitsPerValue, offset);
+                            }
+                            return mul * values.get(doc & mask) + delta;
+                        }
+                    };
+                } else {
+                    final LongValues values = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
+                    if (entry.table != null) {
+                        final long[] table = entry.table;
+                        return new DenseNumericDocValues(maxDoc) {
+                            @Override
+                            public long longValue() throws IOException {
+                                return table[(int) values.get(doc)];
+                            }
+                        };
+                    } else {
+                        final long mul = entry.gcd;
+                        final long delta = entry.minValue;
+                        return new DenseNumericDocValues(maxDoc) {
+                            @Override
+                            public long longValue() throws IOException {
+                                return mul * values.get(doc) + delta;
+                            }
+                        };
+                    }
+                }
+            }
+        } else {
+            // sparse
+            final IndexedDISI disi = new IndexedDISI(data, entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.numValues);
+            if (entry.bitsPerValue == 0) {
+                return new SparseNumericDocValues(disi) {
+                    @Override
+                    public long longValue() throws IOException {
+                        return entry.minValue;
+                    }
+                };
+            } else {
+                final RandomAccessInput slice = data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
+                if (entry.blockShift >= 0) {
+                    // sparse and split into blocks of different bits per value
+                    final int shift = entry.blockShift;
+                    final long mul = entry.gcd;
+                    final int mask = (1 << shift) - 1;
+                    return new SparseNumericDocValues(disi) {
+                        int block = -1;
+                        long delta;
+                        long offset;
+                        long blockEndOffset;
+                        LongValues values;
+
+                        @Override
+                        public long longValue() throws IOException {
+                            final int index = disi.index();
+                            final int block = index >>> shift;
+                            if (this.block != block) {
+                                int bitsPerValue;
+                                do {
+                                    offset = blockEndOffset;
+                                    bitsPerValue = slice.readByte(offset++);
+                                    delta = slice.readLong(offset);
+                                    offset += Long.BYTES;
+                                    if (bitsPerValue == 0) {
+                                        blockEndOffset = offset;
+                                    } else {
+                                        final int length = slice.readInt(offset);
+                                        offset += Integer.BYTES;
+                                        blockEndOffset = offset + length;
+                                    }
+                                    this.block++;
+                                } while (this.block != block);
+                                values = bitsPerValue == 0
+                                    ? LongValues.ZEROES
+                                    : LegacyDirectReader.getInstance(slice, bitsPerValue, offset);
+                            }
+                            return mul * values.get(index & mask) + delta;
+                        }
+                    };
+                } else {
+                    final LongValues values = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
+                    if (entry.table != null) {
+                        final long[] table = entry.table;
+                        return new SparseNumericDocValues(disi) {
+                            @Override
+                            public long longValue() throws IOException {
+                                return table[(int) values.get(disi.index())];
+                            }
+                        };
+                    } else {
+                        final long mul = entry.gcd;
+                        final long delta = entry.minValue;
+                        return new SparseNumericDocValues(disi) {
+                            @Override
+                            public long longValue() throws IOException {
+                                return mul * values.get(disi.index()) + delta;
+                            }
+                        };
+                    }
+                }
+            }
+        }
+    }
+
+    private LongValues getNumericValues(NumericEntry entry) throws IOException {
+        if (entry.bitsPerValue == 0) {
+            return new LongValues() {
+                @Override
+                public long get(long index) {
+                    return entry.minValue;
+                }
+            };
+        } else {
+            final RandomAccessInput slice = data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
+            if (entry.blockShift >= 0) {
+                final int shift = entry.blockShift;
+                final long mul = entry.gcd;
+                final long mask = (1L << shift) - 1;
+                return new LongValues() {
+                    long block = -1;
+                    long delta;
+                    long offset;
+                    long blockEndOffset;
+                    LongValues values;
+
+                    @Override
+                    public long get(long index) {
+                        final long block = index >>> shift;
+                        if (this.block != block) {
+                            assert block > this.block : "Reading backwards is illegal: " + this.block + " < " + block;
+                            int bitsPerValue;
+                            do {
+                                offset = blockEndOffset;
+                                try {
+                                    bitsPerValue = slice.readByte(offset++);
+                                    delta = slice.readLong(offset);
+                                    offset += Long.BYTES;
+                                    if (bitsPerValue == 0) {
+                                        blockEndOffset = offset;
+                                    } else {
+                                        final int length = slice.readInt(offset);
+                                        offset += Integer.BYTES;
+                                        blockEndOffset = offset + length;
+                                    }
+                                } catch (IOException e) {
+                                    throw new RuntimeException(e);
+                                }
+                                this.block++;
+                            } while (this.block != block);
+                            values = bitsPerValue == 0 ? LongValues.ZEROES : LegacyDirectReader.getInstance(slice, bitsPerValue, offset);
+                        }
+                        return mul * values.get(index & mask) + delta;
+                    }
+                };
+            } else {
+                final LongValues values = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
+                if (entry.table != null) {
+                    final long[] table = entry.table;
+                    return new LongValues() {
+                        @Override
+                        public long get(long index) {
+                            return table[(int) values.get(index)];
+                        }
+                    };
+                } else if (entry.gcd != 1) {
+                    final long gcd = entry.gcd;
+                    final long minValue = entry.minValue;
+                    return new LongValues() {
+                        @Override
+                        public long get(long index) {
+                            return values.get(index) * gcd + minValue;
+                        }
+                    };
+                } else if (entry.minValue != 0) {
+                    final long minValue = entry.minValue;
+                    return new LongValues() {
+                        @Override
+                        public long get(long index) {
+                            return values.get(index) + minValue;
+                        }
+                    };
+                } else {
+                    return values;
+                }
+            }
+        }
+    }
+
+    private abstract static class DenseBinaryDocValues extends BinaryDocValues {
+
+        final int maxDoc;
+        int doc = -1;
+
+        DenseBinaryDocValues(int maxDoc) {
+            this.maxDoc = maxDoc;
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            return advance(doc + 1);
+        }
+
+        @Override
+        public int docID() {
+            return doc;
+        }
+
+        @Override
+        public long cost() {
+            return maxDoc;
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            if (target >= maxDoc) {
+                return doc = NO_MORE_DOCS;
+            }
+            return doc = target;
+        }
+
+        @Override
+        public boolean advanceExact(int target) throws IOException {
+            doc = target;
+            return true;
+        }
+    }
+
+    private abstract static class SparseBinaryDocValues extends BinaryDocValues {
+
+        final IndexedDISI disi;
+
+        SparseBinaryDocValues(IndexedDISI disi) {
+            this.disi = disi;
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            return disi.nextDoc();
+        }
+
+        @Override
+        public int docID() {
+            return disi.docID();
+        }
+
+        @Override
+        public long cost() {
+            return disi.cost();
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            return disi.advance(target);
+        }
+
+        @Override
+        public boolean advanceExact(int target) throws IOException {
+            return disi.advanceExact(target);
+        }
+    }
+
+    @Override
+    public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+        BinaryEntry entry = binaries.get(field.name);
+        if (entry.docsWithFieldOffset == -2) {
+            return DocValues.emptyBinary();
+        }
+
+        final IndexInput bytesSlice = data.slice("fixed-binary", entry.dataOffset, entry.dataLength);
+
+        if (entry.docsWithFieldOffset == -1) {
+            // dense
+            if (entry.minLength == entry.maxLength) {
+                // fixed length
+                final int length = entry.maxLength;
+                return new DenseBinaryDocValues(maxDoc) {
+                    final BytesRef bytes = new BytesRef(new byte[length], 0, length);
+
+                    @Override
+                    public BytesRef binaryValue() throws IOException {
+                        bytesSlice.seek((long) doc * length);
+                        bytesSlice.readBytes(bytes.bytes, 0, length);
+                        return bytes;
+                    }
+                };
+            } else {
+                // variable length
+                final RandomAccessInput addressesData = this.data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
+                final LongValues addresses = LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesData);
+                return new DenseBinaryDocValues(maxDoc) {
+                    final BytesRef bytes = new BytesRef(new byte[entry.maxLength], 0, entry.maxLength);
+
+                    @Override
+                    public BytesRef binaryValue() throws IOException {
+                        long startOffset = addresses.get(doc);
+                        bytes.length = (int) (addresses.get(doc + 1L) - startOffset);
+                        bytesSlice.seek(startOffset);
+                        bytesSlice.readBytes(bytes.bytes, 0, bytes.length);
+                        return bytes;
+                    }
+                };
+            }
+        } else {
+            // sparse
+            final IndexedDISI disi = new IndexedDISI(data, entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.numDocsWithField);
+            if (entry.minLength == entry.maxLength) {
+                // fixed length
+                final int length = entry.maxLength;
+                return new SparseBinaryDocValues(disi) {
+                    final BytesRef bytes = new BytesRef(new byte[length], 0, length);
+
+                    @Override
+                    public BytesRef binaryValue() throws IOException {
+                        bytesSlice.seek((long) disi.index() * length);
+                        bytesSlice.readBytes(bytes.bytes, 0, length);
+                        return bytes;
+                    }
+                };
+            } else {
+                // variable length
+                final RandomAccessInput addressesData = this.data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
+                final LongValues addresses = LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesData);
+                return new SparseBinaryDocValues(disi) {
+                    final BytesRef bytes = new BytesRef(new byte[entry.maxLength], 0, entry.maxLength);
+
+                    @Override
+                    public BytesRef binaryValue() throws IOException {
+                        final int index = disi.index();
+                        long startOffset = addresses.get(index);
+                        bytes.length = (int) (addresses.get(index + 1L) - startOffset);
+                        bytesSlice.seek(startOffset);
+                        bytesSlice.readBytes(bytes.bytes, 0, bytes.length);
+                        return bytes;
+                    }
+                };
+            }
+        }
+    }
+
+    @Override
+    public SortedDocValues getSorted(FieldInfo field) throws IOException {
+        SortedEntry entry = sorted.get(field.name);
+        return getSorted(entry);
+    }
+
+    private SortedDocValues getSorted(SortedEntry entry) throws IOException {
+        if (entry.docsWithFieldOffset == -2) {
+            return DocValues.emptySorted();
+        }
+
+        final LongValues ords;
+        if (entry.bitsPerValue == 0) {
+            ords = new LongValues() {
+                @Override
+                public long get(long index) {
+                    return 0L;
+                }
+            };
+        } else {
+            final RandomAccessInput slice = data.randomAccessSlice(entry.ordsOffset, entry.ordsLength);
+            ords = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
+        }
+
+        if (entry.docsWithFieldOffset == -1) {
+            // dense
+            return new BaseSortedDocValues(entry, data) {
+
+                int doc = -1;
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return advance(doc + 1);
+                }
+
+                @Override
+                public int docID() {
+                    return doc;
+                }
+
+                @Override
+                public long cost() {
+                    return maxDoc;
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    if (target >= maxDoc) {
+                        return doc = NO_MORE_DOCS;
+                    }
+                    return doc = target;
+                }
+
+                @Override
+                public boolean advanceExact(int target) {
+                    doc = target;
+                    return true;
+                }
+
+                @Override
+                public int ordValue() {
+                    return (int) ords.get(doc);
+                }
+            };
+        } else {
+            // sparse
+            final IndexedDISI disi = new IndexedDISI(data, entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.numDocsWithField);
+            return new BaseSortedDocValues(entry, data) {
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return disi.nextDoc();
+                }
+
+                @Override
+                public int docID() {
+                    return disi.docID();
+                }
+
+                @Override
+                public long cost() {
+                    return disi.cost();
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    return disi.advance(target);
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    return disi.advanceExact(target);
+                }
+
+                @Override
+                public int ordValue() {
+                    return (int) ords.get(disi.index());
+                }
+            };
+        }
+    }
+
+    private abstract static class BaseSortedDocValues extends SortedDocValues {
+
+        final SortedEntry entry;
+        final IndexInput data;
+        final TermsEnum termsEnum;
+
+        BaseSortedDocValues(SortedEntry entry, IndexInput data) throws IOException {
+            this.entry = entry;
+            this.data = data;
+            this.termsEnum = termsEnum();
+        }
+
+        @Override
+        public int getValueCount() {
+            return Math.toIntExact(entry.termsDictSize);
+        }
+
+        @Override
+        public BytesRef lookupOrd(int ord) throws IOException {
+            termsEnum.seekExact(ord);
+            return termsEnum.term();
+        }
+
+        @Override
+        public int lookupTerm(BytesRef key) throws IOException {
+            SeekStatus status = termsEnum.seekCeil(key);
+            switch (status) {
+                case FOUND:
+                    return Math.toIntExact(termsEnum.ord());
+                case NOT_FOUND:
+                case END:
+                default:
+                    return Math.toIntExact(-1L - termsEnum.ord());
+            }
+        }
+
+        @Override
+        public TermsEnum termsEnum() throws IOException {
+            return new TermsDict(entry, data);
+        }
+    }
+
+    private abstract static class BaseSortedSetDocValues extends SortedSetDocValues {
+
+        final SortedSetEntry entry;
+        final IndexInput data;
+        final TermsEnum termsEnum;
+
+        BaseSortedSetDocValues(SortedSetEntry entry, IndexInput data) throws IOException {
+            this.entry = entry;
+            this.data = data;
+            this.termsEnum = termsEnum();
+        }
+
+        @Override
+        public long getValueCount() {
+            return entry.termsDictSize;
+        }
+
+        @Override
+        public BytesRef lookupOrd(long ord) throws IOException {
+            termsEnum.seekExact(ord);
+            return termsEnum.term();
+        }
+
+        @Override
+        public long lookupTerm(BytesRef key) throws IOException {
+            SeekStatus status = termsEnum.seekCeil(key);
+            switch (status) {
+                case FOUND:
+                    return termsEnum.ord();
+                case NOT_FOUND:
+                case END:
+                default:
+                    return -1L - termsEnum.ord();
+            }
+        }
+
+        @Override
+        public TermsEnum termsEnum() throws IOException {
+            return new TermsDict(entry, data);
+        }
+    }
+
+    private static class TermsDict extends BaseTermsEnum {
+
+        final TermsDictEntry entry;
+        final LongValues blockAddresses;
+        final IndexInput bytes;
+        final long blockMask;
+        final LongValues indexAddresses;
+        final IndexInput indexBytes;
+        final BytesRef term;
+        long ord = -1;
+
+        TermsDict(TermsDictEntry entry, IndexInput data) throws IOException {
+            this.entry = entry;
+            RandomAccessInput addressesSlice = data.randomAccessSlice(entry.termsAddressesOffset, entry.termsAddressesLength);
+            blockAddresses = LegacyDirectMonotonicReader.getInstance(entry.termsAddressesMeta, addressesSlice);
+            bytes = data.slice("terms", entry.termsDataOffset, entry.termsDataLength);
+            blockMask = (1L << entry.termsDictBlockShift) - 1;
+            RandomAccessInput indexAddressesSlice = data.randomAccessSlice(
+                entry.termsIndexAddressesOffset,
+                entry.termsIndexAddressesLength
+            );
+            indexAddresses = LegacyDirectMonotonicReader.getInstance(entry.termsIndexAddressesMeta, indexAddressesSlice);
+            indexBytes = data.slice("terms-index", entry.termsIndexOffset, entry.termsIndexLength);
+            term = new BytesRef(entry.maxTermLength);
+        }
+
+        @Override
+        public BytesRef next() throws IOException {
+            if (++ord >= entry.termsDictSize) {
+                return null;
+            }
+            if ((ord & blockMask) == 0L) {
+                term.length = bytes.readVInt();
+                bytes.readBytes(term.bytes, 0, term.length);
+            } else {
+                final int token = Byte.toUnsignedInt(bytes.readByte());
+                int prefixLength = token & 0x0F;
+                int suffixLength = 1 + (token >>> 4);
+                if (prefixLength == 15) {
+                    prefixLength += bytes.readVInt();
+                }
+                if (suffixLength == 16) {
+                    suffixLength += bytes.readVInt();
+                }
+                term.length = prefixLength + suffixLength;
+                bytes.readBytes(term.bytes, prefixLength, suffixLength);
+            }
+            return term;
+        }
+
+        @Override
+        public void seekExact(long ord) throws IOException {
+            if (ord < 0 || ord >= entry.termsDictSize) {
+                throw new IndexOutOfBoundsException();
+            }
+            final long blockIndex = ord >>> entry.termsDictBlockShift;
+            final long blockAddress = blockAddresses.get(blockIndex);
+            bytes.seek(blockAddress);
+            this.ord = (blockIndex << entry.termsDictBlockShift) - 1;
+            do {
+                next();
+            } while (this.ord < ord);
+        }
+
+        private BytesRef getTermFromIndex(long index) throws IOException {
+            assert index >= 0 && index <= (entry.termsDictSize - 1) >>> entry.termsDictIndexShift;
+            final long start = indexAddresses.get(index);
+            term.length = (int) (indexAddresses.get(index + 1) - start);
+            indexBytes.seek(start);
+            indexBytes.readBytes(term.bytes, 0, term.length);
+            return term;
+        }
+
+        private long seekTermsIndex(BytesRef text) throws IOException {
+            long lo = 0L;
+            long hi = (entry.termsDictSize - 1) >>> entry.termsDictIndexShift;
+            while (lo <= hi) {
+                final long mid = (lo + hi) >>> 1;
+                getTermFromIndex(mid);
+                final int cmp = term.compareTo(text);
+                if (cmp <= 0) {
+                    lo = mid + 1;
+                } else {
+                    hi = mid - 1;
+                }
+            }
+
+            assert hi < 0 || getTermFromIndex(hi).compareTo(text) <= 0;
+            assert hi == ((entry.termsDictSize - 1) >>> entry.termsDictIndexShift) || getTermFromIndex(hi + 1).compareTo(text) > 0;
+
+            return hi;
+        }
+
+        private BytesRef getFirstTermFromBlock(long block) throws IOException {
+            assert block >= 0 && block <= (entry.termsDictSize - 1) >>> entry.termsDictBlockShift;
+            final long blockAddress = blockAddresses.get(block);
+            bytes.seek(blockAddress);
+            term.length = bytes.readVInt();
+            bytes.readBytes(term.bytes, 0, term.length);
+            return term;
+        }
+
+        private long seekBlock(BytesRef text) throws IOException {
+            long index = seekTermsIndex(text);
+            if (index == -1L) {
+                return -1L;
+            }
+
+            long ordLo = index << entry.termsDictIndexShift;
+            long ordHi = Math.min(entry.termsDictSize, ordLo + (1L << entry.termsDictIndexShift)) - 1L;
+
+            long blockLo = ordLo >>> entry.termsDictBlockShift;
+            long blockHi = ordHi >>> entry.termsDictBlockShift;
+
+            while (blockLo <= blockHi) {
+                final long blockMid = (blockLo + blockHi) >>> 1;
+                getFirstTermFromBlock(blockMid);
+                final int cmp = term.compareTo(text);
+                if (cmp <= 0) {
+                    blockLo = blockMid + 1;
+                } else {
+                    blockHi = blockMid - 1;
+                }
+            }
+
+            assert blockHi < 0 || getFirstTermFromBlock(blockHi).compareTo(text) <= 0;
+            assert blockHi == ((entry.termsDictSize - 1) >>> entry.termsDictBlockShift)
+                || getFirstTermFromBlock(blockHi + 1).compareTo(text) > 0;
+
+            return blockHi;
+        }
+
+        @Override
+        public SeekStatus seekCeil(BytesRef text) throws IOException {
+            final long block = seekBlock(text);
+            if (block == -1) {
+                // before the first term
+                seekExact(0L);
+                return SeekStatus.NOT_FOUND;
+            }
+            final long blockAddress = blockAddresses.get(block);
+            this.ord = block << entry.termsDictBlockShift;
+            bytes.seek(blockAddress);
+            term.length = bytes.readVInt();
+            bytes.readBytes(term.bytes, 0, term.length);
+            while (true) {
+                int cmp = term.compareTo(text);
+                if (cmp == 0) {
+                    return SeekStatus.FOUND;
+                } else if (cmp > 0) {
+                    return SeekStatus.NOT_FOUND;
+                }
+                if (next() == null) {
+                    return SeekStatus.END;
+                }
+            }
+        }
+
+        @Override
+        public BytesRef term() throws IOException {
+            return term;
+        }
+
+        @Override
+        public long ord() throws IOException {
+            return ord;
+        }
+
+        @Override
+        public long totalTermFreq() throws IOException {
+            return -1L;
+        }
+
+        @Override
+        public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ImpactsEnum impacts(int flags) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int docFreq() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    @Override
+    public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+        SortedNumericEntry entry = sortedNumerics.get(field.name);
+        if (entry.numValues == entry.numDocsWithField) {
+            return DocValues.singleton(getNumeric(entry));
+        }
+
+        final RandomAccessInput addressesInput = data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
+        final LongValues addresses = LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesInput);
+
+        final LongValues values = getNumericValues(entry);
+
+        if (entry.docsWithFieldOffset == -1) {
+            // dense
+            return new SortedNumericDocValues() {
+
+                int doc = -1;
+                long start, end;
+                int count;
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return advance(doc + 1);
+                }
+
+                @Override
+                public int docID() {
+                    return doc;
+                }
+
+                @Override
+                public long cost() {
+                    return maxDoc;
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    if (target >= maxDoc) {
+                        return doc = NO_MORE_DOCS;
+                    }
+                    start = addresses.get(target);
+                    end = addresses.get(target + 1L);
+                    count = (int) (end - start);
+                    return doc = target;
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    start = addresses.get(target);
+                    end = addresses.get(target + 1L);
+                    count = (int) (end - start);
+                    doc = target;
+                    return true;
+                }
+
+                @Override
+                public long nextValue() throws IOException {
+                    return values.get(start++);
+                }
+
+                @Override
+                public int docValueCount() {
+                    return count;
+                }
+            };
+        } else {
+            // sparse
+            final IndexedDISI disi = new IndexedDISI(data, entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.numDocsWithField);
+            return new SortedNumericDocValues() {
+
+                boolean set;
+                long start, end;
+                int count;
+
+                @Override
+                public int nextDoc() throws IOException {
+                    set = false;
+                    return disi.nextDoc();
+                }
+
+                @Override
+                public int docID() {
+                    return disi.docID();
+                }
+
+                @Override
+                public long cost() {
+                    return disi.cost();
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    set = false;
+                    return disi.advance(target);
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    set = false;
+                    return disi.advanceExact(target);
+                }
+
+                @Override
+                public long nextValue() throws IOException {
+                    set();
+                    return values.get(start++);
+                }
+
+                @Override
+                public int docValueCount() {
+                    set();
+                    return count;
+                }
+
+                private void set() {
+                    if (set == false) {
+                        final int index = disi.index();
+                        start = addresses.get(index);
+                        end = addresses.get(index + 1L);
+                        count = (int) (end - start);
+                        set = true;
+                    }
+                }
+            };
+        }
+    }
+
+    @Override
+    public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+        SortedSetEntry entry = sortedSets.get(field.name);
+        if (entry.singleValueEntry != null) {
+            return DocValues.singleton(getSorted(entry.singleValueEntry));
+        }
+
+        final RandomAccessInput slice = data.randomAccessSlice(entry.ordsOffset, entry.ordsLength);
+        final LongValues ords = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
+
+        final RandomAccessInput addressesInput = data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
+        final LongValues addresses = LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesInput);
+
+        if (entry.docsWithFieldOffset == -1) {
+            // dense
+            return new BaseSortedSetDocValues(entry, data) {
+
+                int doc = -1;
+                long start, end;
+                int count;
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return advance(doc + 1);
+                }
+
+                @Override
+                public int docID() {
+                    return doc;
+                }
+
+                @Override
+                public long cost() {
+                    return maxDoc;
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    if (target >= maxDoc) {
+                        return doc = NO_MORE_DOCS;
+                    }
+                    start = addresses.get(target);
+                    end = addresses.get(target + 1L);
+                    count = (int) (end - start);
+                    return doc = target;
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    start = addresses.get(target);
+                    end = addresses.get(target + 1L);
+                    count = (int) (end - start);
+                    doc = target;
+                    return true;
+                }
+
+                @Override
+                public long nextOrd() throws IOException {
+                    if (start == end) {
+                        return NO_MORE_ORDS;
+                    }
+                    return ords.get(start++);
+                }
+
+                @Override
+                public int docValueCount() {
+                    return count;
+                }
+            };
+        } else {
+            // sparse
+            final IndexedDISI disi = new IndexedDISI(data, entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.numDocsWithField);
+            return new BaseSortedSetDocValues(entry, data) {
+
+                boolean set;
+                long start;
+                long end = 0;
+                int count;
+
+                @Override
+                public int nextDoc() throws IOException {
+                    set = false;
+                    return disi.nextDoc();
+                }
+
+                @Override
+                public int docID() {
+                    return disi.docID();
+                }
+
+                @Override
+                public long cost() {
+                    return disi.cost();
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    set = false;
+                    return disi.advance(target);
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    set = false;
+                    return disi.advanceExact(target);
+                }
+
+                private boolean set() {
+                    if (set == false) {
+                        final int index = disi.index();
+                        start = addresses.get(index);
+                        end = addresses.get(index + 1L);
+                        count = (int) (end - start);
+                        set = true;
+                        return true;
+                    }
+                    return false;
+                }
+
+                @Override
+                public long nextOrd() throws IOException {
+                    if (set()) {
+                        return ords.get(start++);
+                    } else if (start == end) {
+                        return NO_MORE_ORDS;
+                    } else {
+                        return ords.get(start++);
+                    }
+                }
+
+                @Override
+                public int docValueCount() {
+                    set();
+                    return count;
+                }
+            };
+        }
+    }
+
+    @Override
+    public void checkIntegrity() throws IOException {
+        CodecUtil.checksumEntireFile(data);
+    }
+
+    @Override
+    public DocValuesSkipper getSkipper(FieldInfo field) {
+        return null;
+    }
+}
diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 6e5205d664f2d..0215e9f7ca4ab 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -6,5 +6,6 @@
 #
 
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60Codec
diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
index 2d46b4bca3d0c..8d24d86982da8 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
+++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
@@ -14,3 +14,4 @@
 #  limitations under the License.
 
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesFormat
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70DocValuesFormat
diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java
index f62ab9fbc4fee..0b72b96b446d4 100644
--- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java
+++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java
@@ -18,7 +18,7 @@ public class OldCodecsAvailableTests extends ESTestCase {
      * to the next major Lucene version.
      */
     @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS)
-    @AwaitsFix(bugUrl = "muted until we add bwc codecs as part of lucene 10 upgrade")
+    @AwaitsFix(bugUrl = "muted until we add bwc codecs to support 7.x indices in Elasticsearch 9.0")
     public void testLuceneBWCCodecsAvailable() {
         assertEquals("Add Lucene BWC codecs for Elasticsearch version 7", 8, Version.CURRENT.major);
     }
diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/BlockPostingsFormat3Tests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/BlockPostingsFormat3Tests.java
index 304f7b0c934fb..59f5e5de1eff7 100644
--- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/BlockPostingsFormat3Tests.java
+++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/BlockPostingsFormat3Tests.java
@@ -48,7 +48,9 @@
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.test.ESTestCase;
 
@@ -187,7 +189,11 @@ public void assertTerms(Terms leftTerms, Terms rightTerms, boolean deep) throws
             int numIntersections = atLeast(3);
             for (int i = 0; i < numIntersections; i++) {
                 String re = AutomatonTestUtil.randomRegexp(random());
-                CompiledAutomaton automaton = new CompiledAutomaton(new RegExp(re, RegExp.NONE).toAutomaton());
+                Automaton determinized = Operations.determinize(
+                    new RegExp(re, RegExp.NONE).toAutomaton(),
+                    Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
+                );
+                CompiledAutomaton automaton = new CompiledAutomaton(determinized);
                 if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
                     // TODO: test start term too
                     TermsEnum leftIntersection = leftTerms.intersect(automaton, null);
diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java
index c819dca3ec6ff..1a2aca0d63bde 100644
--- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java
+++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesFormatTests.java
@@ -10,12 +10,12 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.index.LegacyBaseDocValuesFormatTestCase;
 import org.apache.lucene.tests.util.TestUtil;
 import org.elasticsearch.test.GraalVMThreadsFilter;
 
 @ThreadLeakFilters(filters = { GraalVMThreadsFilter.class })
-public class Lucene54DocValuesFormatTests extends BaseDocValuesFormatTestCase {
+public class Lucene54DocValuesFormatTests extends LegacyBaseDocValuesFormatTestCase {
 
     private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene54DocValuesFormat());
 
diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesFormatTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesFormatTests.java
new file mode 100644
index 0000000000000..ce645feb854d1
--- /dev/null
+++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/Lucene70DocValuesFormatTests.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.tests.index.LegacyBaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.elasticsearch.test.GraalVMThreadsFilter;
+
+@ThreadLeakFilters(filters = { GraalVMThreadsFilter.class })
+public class Lucene70DocValuesFormatTests extends LegacyBaseDocValuesFormatTestCase {
+
+    private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene70DocValuesFormat());
+
+    @Override
+    protected Codec getCodec() {
+        return codec;
+    }
+}
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml
index 37dd93b7f16d9..d2df9b5629704 100644
--- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml
@@ -3,6 +3,8 @@ _meta:
   description: Default mappings for the OpenTelemetry metrics index template installed by x-pack
   managed: true
 template:
+  settings:
+    index.mapping.ignore_malformed: true
   mappings:
     properties:
       start_timestamp:
@@ -19,27 +21,22 @@ template:
       - histogram:
           mapping:
             type: histogram
-            ignore_malformed: true
       - counter_long:
           mapping:
             type: long
             time_series_metric: counter
-            ignore_malformed: true
       - gauge_long:
           mapping:
             type: long
             time_series_metric: gauge
-            ignore_malformed: true
       - counter_double:
           mapping:
             type: double
             time_series_metric: counter
-            ignore_malformed: true
       - gauge_double:
           mapping:
             type: double
             time_series_metric: gauge
-            ignore_malformed: true
       - summary:
           mapping:
             type: aggregate_metric_double
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml
index 513e1a857787e..4a039886ecc4e 100644
--- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml
@@ -17,6 +17,9 @@ template:
         type: constant_keyword
       data_stream.namespace:
         type: constant_keyword
+      event.dataset:
+        type: alias
+        path: data_stream.dataset
       attributes:
         type: passthrough
         dynamic: true
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml
index 6645e7d282520..eb5cd6d37af83 100644
--- a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml
@@ -56,21 +56,45 @@ template:
               os.version:
                 type: keyword
                 ignore_above: 1024
+              k8s.container.name:
+                type: keyword
+                ignore_above: 1024
+              k8s.cronjob.name:
+                type: keyword
+                ignore_above: 1024
+              k8s.daemonset.name:
+                type: keyword
+                ignore_above: 1024
               k8s.deployment.name:
                 type: keyword
                 ignore_above: 1024
+              k8s.job.name:
+                type: keyword
+                ignore_above: 1024
               k8s.namespace.name:
                 type: keyword
                 ignore_above: 1024
+              k8s.node.hostname:
+                type: keyword
+                ignore_above: 1024
               k8s.node.name:
                 type: keyword
                 ignore_above: 1024
+              k8s.node.uid:
+                type: keyword
+                ignore_above: 1024
               k8s.pod.name:
                 type: keyword
                 ignore_above: 1024
               k8s.pod.uid:
                 type: keyword
                 ignore_above: 1024
+              k8s.replicaset.name:
+                type: keyword
+                ignore_above: 1024
+              k8s.statefulset.name:
+                type: keyword
+                ignore_above: 1024
       service.node.name:
         type: alias
         path: resource.attributes.service.instance.id
@@ -122,6 +146,30 @@ template:
       kubernetes.pod.uid:
         type: alias
         path: resource.attributes.k8s.pod.uid
+      kubernetes.container.name:
+        type: alias
+        path: resource.attributes.k8s.container.name
+      kubernetes.cronjob.name:
+        type: alias
+        path: resource.attributes.k8s.cronjob.name
+      kubernetes.job.name:
+        type: alias
+        path: resource.attributes.k8s.job.name
+      kubernetes.statefulset.name:
+        type: alias
+        path: resource.attributes.k8s.statefulset.name
+      kubernetes.daemonset.name:
+        type: alias
+        path: resource.attributes.k8s.daemonset.name
+      kubernetes.replicaset.name:
+        type: alias
+        path: resource.attributes.k8s.replicaset.name
+      kubernetes.node.uid:
+        type: alias
+        path: resource.attributes.k8s.node.uid
+      kubernetes.node.hostname:
+        type: alias
+        path: resource.attributes.k8s.node.hostname
 # Below are non-ECS fields that may be used by Kibana.
       service.language.name:
         type: alias
diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml
index 3b4c3127bb71c..f8489605ad1bf 100644
--- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml
+++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml
@@ -27,14 +27,3 @@ template:
       data_stream.type:
         type: constant_keyword
         value: metrics
-    dynamic_templates:
-      - ecs_ip:
-          mapping:
-            type: ip
-          path_match: [ "ip", "*.ip", "*_ip" ]
-          match_mapping_type: string
-      - all_strings_to_keywords:
-          mapping:
-            ignore_above: 1024
-            type: keyword
-          match_mapping_type: string
diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml
index 52873287696ab..b2d30c7f85cc4 100644
--- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml
+++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml
@@ -1,7 +1,7 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin otel-data. This must be increased whenever an existing template is
 # changed, in order for it to be updated on Elasticsearch upgrade.
-version: 5
+version: 6
 
 component-templates:
   - otel@mappings
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml
index 0957a79552ad3..63966e601a3cb 100644
--- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml
@@ -145,3 +145,82 @@ Structured log body:
         index: $datastream-backing-index
   - is_true: $datastream-backing-index
   - match: { .$datastream-backing-index.mappings.properties.body.properties.flattened.type: "flattened" }
+---
+"event.dataset alias must point to data_stream.dataset":
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}'
+  - is_false: errors
+  - is_false: errors
+  - do:
+      search:
+        index: logs-generic.otel-default
+        body:
+          fields: ["event.dataset"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.event\.dataset: ["generic.otel"] }
+---
+host.name pass-through:
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - "@timestamp": 2024-07-18T14:48:33.467654000Z
+            resource:
+              attributes:
+                host.name: localhost
+  - is_false: errors
+  - do:
+      search:
+        index: logs-generic.otel-default
+        body:
+          query:
+            term:
+              host.name: localhost
+          fields: [ "*" ]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.resource\.attributes\.host\.name: [ "localhost" ] }
+  - match: { hits.hits.0.fields.host\.name: [ "localhost" ] }
+---
+"kubernetes.* -> resource.attributes.k8s.* aliases":
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: { }
+          - "@timestamp": 2024-07-18T14:48:33.467654000Z
+            data_stream:
+              dataset: generic.otel
+              namespace: default
+            resource:
+              attributes:
+                k8s.container.name: myContainerName
+                k8s.cronjob.name: myCronJobName
+                k8s.job.name: myJobName
+                k8s.statefulset.name: myStatefulsetName
+                k8s.daemonset.name: myDaemonsetName
+                k8s.replicaset.name: myReplicasetName
+                k8s.node.uid: myNodeUid
+                k8s.node.hostname: myNodeHostname
+  - is_false: errors
+  - do:
+      search:
+        index: logs-generic.otel-default
+        body:
+          fields: ["kubernetes.container.name", "kubernetes.cronjob.name", "kubernetes.job.name", "kubernetes.statefulset.name", "kubernetes.daemonset.name", "kubernetes.replicaset.name", "kubernetes.node.uid", "kubernetes.node.hostname" ]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.kubernetes\.container\.name : ["myContainerName"] }
+  - match: { hits.hits.0.fields.kubernetes\.cronjob\.name : ["myCronJobName"] }
+  - match: { hits.hits.0.fields.kubernetes\.job\.name : ["myJobName"] }
+  - match: { hits.hits.0.fields.kubernetes\.statefulset\.name : ["myStatefulsetName"] }
+  - match: { hits.hits.0.fields.kubernetes\.daemonset\.name : ["myDaemonsetName"] }
+  - match: { hits.hits.0.fields.kubernetes\.replicaset\.name : ["myReplicasetName"] }
+  - match: { hits.hits.0.fields.kubernetes\.node\.uid : ["myNodeUid"] }
+  - match: { hits.hits.0.fields.kubernetes\.node\.hostname : ["myNodeHostname"] }
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
index 81aded9cef3be..1aafe3765813b 100644
--- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
@@ -245,3 +245,37 @@ IP dimensions:
   - match: { .$idx0name.mappings.properties.metrics.properties.summary.type: 'aggregate_metric_double' }
   - match: { .$idx0name.mappings.properties.metrics.properties.summary_minmax.type: 'aggregate_metric_double' }
   - match: { .$idx0name.mappings.properties.metrics.properties.histogram.type: 'histogram' }
+---
+Empty IP field:
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {"dynamic_templates":{"metrics.foo.bar":"counter_long"}}
+          - "@timestamp": 2024-07-18T14:48:33.467654000Z
+            resource:
+              attributes:
+                host.name: localhost
+                host.ip: ""
+            metrics:
+              foo.bar: 42
+  - is_false: errors
+
+  - do:
+      indices.get_data_stream:
+        name: metrics-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+
+  - do:
+      indices.get_mapping:
+        index: $idx0name
+        expand_wildcards: hidden
+  - match: { .$idx0name.mappings.properties.resource.properties.attributes.properties.host\.ip.type: 'ip' }
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["*"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._ignored: ["resource.attributes.host.ip"] }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java
index 24f2f287f4cd6..5d32c39e350a5 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java
@@ -11,12 +11,13 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -176,89 +177,56 @@ public long getTotalSamples() {
     @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // change casing from Camel Case to Snake Case (requires updates in Kibana as well)
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return Iterators.concat(
-            ChunkedToXContentHelper.startObject(),
-            ChunkedToXContentHelper.array(
-                "Edges",
-                Iterators.flatMap(
-                    edges.iterator(),
-                    perNodeEdges -> Iterators.concat(
-                        ChunkedToXContentHelper.startArray(),
-                        Iterators.map(perNodeEdges.entrySet().iterator(), edge -> (b, p) -> b.value(edge.getValue())),
-                        ChunkedToXContentHelper.endArray()
-                    )
-                )
-            ),
-            ChunkedToXContentHelper.array("FileID", Iterators.map(fileIds.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("FrameType", Iterators.map(frameTypes.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("Inline", Iterators.map(inlineFrames.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("ExeFilename", Iterators.map(fileNames.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("AddressOrLine", Iterators.map(addressOrLines.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("FunctionName", Iterators.map(functionNames.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("FunctionOffset");
-                for (int functionOffset : functionOffsets) {
-                    b.value(functionOffset);
-                }
-                return b.endArray();
-            }),
-            ChunkedToXContentHelper.array("SourceFilename", Iterators.map(sourceFileNames.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("SourceLine");
-                for (int sourceLine : sourceLines) {
-                    b.value(sourceLine);
-                }
-                return b.endArray();
-            }),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("CountInclusive");
-                for (long countInclusive : countInclusive) {
-                    b.value(countInclusive);
-                }
-                return b.endArray();
-            }),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("CountExclusive");
-                for (long c : countExclusive) {
-                    b.value(c);
-                }
-                return b.endArray();
-            }),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("AnnualCO2TonsInclusive");
-                for (double co2Tons : annualCO2TonsInclusive) {
-                    // write as raw value - we need direct control over the output representation (here: limit to 4 decimal places)
-                    b.rawValue(NumberUtils.doubleToString(co2Tons));
-                }
-                return b.endArray();
-            }),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("AnnualCO2TonsExclusive");
-                for (double co2Tons : annualCO2TonsExclusive) {
-                    b.rawValue(NumberUtils.doubleToString(co2Tons));
-                }
-                return b.endArray();
-            }),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("AnnualCostsUSDInclusive");
-                for (double costs : annualCostsUSDInclusive) {
-                    b.rawValue(NumberUtils.doubleToString(costs));
-                }
-                return b.endArray();
-            }),
-            ChunkedToXContentHelper.singleChunk((b, p) -> {
-                b.startArray("AnnualCostsUSDExclusive");
-                for (double costs : annualCostsUSDExclusive) {
-                    b.rawValue(NumberUtils.doubleToString(costs));
-                }
-                return b.endArray();
-            }),
-            Iterators.single((b, p) -> b.field("Size", size)),
-            Iterators.single((b, p) -> b.field("SamplingRate", samplingRate)),
-            Iterators.single((b, p) -> b.field("SelfCPU", selfCPU)),
-            Iterators.single((b, p) -> b.field("TotalCPU", totalCPU)),
-            Iterators.single((b, p) -> b.field("TotalSamples", totalSamples)),
-            ChunkedToXContentHelper.endObject()
-        );
+        return ChunkedToXContent.builder(params).object(ob -> {
+            ob.array("Edges", edges.iterator(), (eb, e) -> eb.array(intValues(e.values())));
+            ob.array("FileID", fileIds.toArray(String[]::new));
+            ob.array("FrameType", intValues(frameTypes));
+            ob.array("Inline", inlineFrames.iterator(), e -> (b, p) -> b.value(e));
+            ob.array("ExeFilename", fileNames.toArray(String[]::new));
+            ob.array("AddressOrLine", intValues(addressOrLines));
+            ob.array("FunctionName", functionNames.toArray(String[]::new));
+            ob.array("FunctionOffset", intValues(functionOffsets));
+            ob.array("SourceFilename", sourceFileNames.toArray(String[]::new));
+            ob.array("SourceLine", intValues(sourceLines));
+            ob.array("CountInclusive", longValues(countInclusive));
+            ob.array("CountExclusive", longValues(countExclusive));
+            ob.array("AnnualCO2TonsInclusive", doubleValues(annualCO2TonsInclusive));
+            ob.array("AnnualCO2TonsExclusive", doubleValues(annualCO2TonsExclusive));
+            ob.array("AnnualCostsUSDInclusive", doubleValues(annualCostsUSDInclusive));
+            ob.array("AnnualCostsUSDExclusive", doubleValues(annualCostsUSDExclusive));
+            ob.field("Size", size);
+            ob.field("SamplingRate", samplingRate);
+            ob.field("SelfCPU", selfCPU);
+            ob.field("TotalCPU", totalCPU);
+            ob.field("TotalSamples", totalSamples);
+        });
+    }
+
+    private static Iterator<ToXContent> intValues(Collection<Integer> values) {
+        return Iterators.single((b, p) -> {
+            for (Integer i : values) {
+                b.value(i);
+            }
+            return b;
+        });
+    }
+
+    private static Iterator<ToXContent> longValues(Collection<Long> values) {
+        return Iterators.single((b, p) -> {
+            for (Long l : values) {
+                b.value(l);
+            }
+            return b;
+        });
+    }
+
+    private static Iterator<ToXContent> doubleValues(Collection<Double> values) {
+        return Iterators.single((b, p) -> {
+            for (Double d : values) {
+                // write as raw value - we need direct control over the output representation (here: limit to 4 decimal places)
+                b.rawValue(NumberUtils.doubleToString(d));
+            }
+            return b;
+        });
     }
 }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
index 48673d2002170..f447f67b4cdd2 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
@@ -179,7 +179,7 @@ private void searchProfilingEvents(
             .setQuery(request.getQuery())
             .setTrackTotalHits(true)
             .execute(ActionListener.wrap(searchResponse -> {
-                long sampleCount = searchResponse.getHits().getTotalHits().value;
+                long sampleCount = searchResponse.getHits().getTotalHits().value();
                 EventsIndex resampledIndex = mediumDownsampled.getResampledIndex(request.getSampleSize(), sampleCount);
                 log.debug(
                     "User requested [{}] samples, [{}] samples matched in [{}]. Picking [{}]",
@@ -220,7 +220,7 @@ private void searchGenericEvents(
             .setPreference(String.valueOf(request.hashCode()))
             .setQuery(request.getQuery())
             .execute(ActionListener.wrap(searchResponse -> {
-                long sampleCount = searchResponse.getHits().getTotalHits().value;
+                long sampleCount = searchResponse.getHits().getTotalHits().value();
                 int requestedSampleCount = request.getSampleSize();
                 // random sampler aggregation does not support sampling rates between 0.5 and 1.0 -> clamp to 1.0
                 if (sampleCount <= requestedSampleCount * 2L) {
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java
index 9dd46e778fb9a..dbb4cf4dc6856 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java
@@ -180,7 +180,7 @@ private void execute(ClusterState state, ActionListener<GetStatusAction.Response
                 countRequest.source(searchSourceBuilder);
 
                 nodeClient.search(countRequest, ActionListener.wrap(searchResponse -> {
-                    boolean hasData = searchResponse.getHits().getTotalHits().value > 0;
+                    boolean hasData = searchResponse.getHits().getTotalHits().value() > 0;
                     listener.onResponse(
                         new GetStatusAction.Response(pluginEnabled, resourceManagementEnabled, resourcesCreated, anyPre891Data, hasData)
                     );
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java
index 3943ddd3e207a..73e47a631de96 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java
@@ -121,9 +121,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("is_running", isRunning);
             builder.field("is_partial", isPartial);
             if (startTimeMillis != null) { // start time is available only for a running eql search
-                builder.timeField("start_time_in_millis", "start_time", startTimeMillis);
+                builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTimeMillis);
             }
-            builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
+            builder.timestampFieldsFromUnixEpochMillis("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
             if (isRunning == false) { // completion status is available only for a completed eql search
                 builder.field("completion_status", completionStatus.getStatus());
             }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
index 8eac03d36371e..e4f5810ac89d3 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
@@ -9,7 +9,6 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
@@ -51,8 +50,7 @@ public char escape() {
 
     @Override
     public Automaton createAutomaton() {
-        Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard));
-        return MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        return WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
index 528872ca9b4cf..41ae97ec5e4fd 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
 import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 
 import java.util.Objects;
@@ -21,7 +22,10 @@ public RLikePattern(String regexpPattern) {
 
     @Override
     public Automaton createAutomaton() {
-        return new RegExp(regexpPattern).toAutomaton();
+        return Operations.determinize(
+            new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton(),
+            Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/WildcardPattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/WildcardPattern.java
index fd6bd177e4c60..6703f1aeacbb5 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/WildcardPattern.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/WildcardPattern.java
@@ -9,7 +9,6 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
@@ -39,8 +38,7 @@ public String pattern() {
 
     @Override
     public Automaton createAutomaton() {
-        Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard));
-        return MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        return WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java
deleted file mode 100644
index 6431c83ee1c2e..0000000000000
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.ql.plugin;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.action.search.VersionMismatchException;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.xpack.ql.util.Holder;
-
-import java.util.function.Consumer;
-
-public final class TransportActionUtils {
-
-    /**
-     * Execute a *QL request and re-try it in case the first request failed with a {@code VersionMismatchException}
-     *
-     * @param clusterService The cluster service instance
-     * @param onFailure On-failure handler in case the request doesn't fail with a {@code VersionMismatchException}
-     * @param queryRunner *QL query execution code, typically a Plan Executor running the query
-     * @param retryRequest Re-trial logic
-     * @param log Log4j logger
-     */
-    public static void executeRequestWithRetryAttempt(
-        ClusterService clusterService,
-        Consumer<Exception> onFailure,
-        Consumer<Consumer<Exception>> queryRunner,
-        Consumer<DiscoveryNode> retryRequest,
-        Logger log
-    ) {
-
-        Holder<Boolean> retrySecondTime = new Holder<Boolean>(false);
-        queryRunner.accept(e -> {
-            // the search request likely ran on nodes with different versions of ES
-            // we will retry on a node with an older version that should generate a backwards compatible _search request
-            if (e instanceof SearchPhaseExecutionException
-                && ((SearchPhaseExecutionException) e).getCause() instanceof VersionMismatchException) {
-                if (log.isDebugEnabled()) {
-                    log.debug("Caught exception type [{}] with cause [{}].", e.getClass().getName(), e.getCause());
-                }
-                DiscoveryNode localNode = clusterService.state().nodes().getLocalNode();
-                DiscoveryNode candidateNode = null;
-                for (DiscoveryNode node : clusterService.state().nodes()) {
-                    // find the first node that's older than the current node
-                    if (node != localNode && node.getVersion().before(localNode.getVersion())) {
-                        candidateNode = node;
-                        break;
-                    }
-                }
-                if (candidateNode != null) {
-                    if (log.isDebugEnabled()) {
-                        log.debug(
-                            "Candidate node to resend the request to: address [{}], id [{}], name [{}], version [{}]",
-                            candidateNode.getAddress(),
-                            candidateNode.getId(),
-                            candidateNode.getName(),
-                            candidateNode.getVersion()
-                        );
-                    }
-                    // re-send the request to the older node
-                    retryRequest.accept(candidateNode);
-                } else {
-                    retrySecondTime.set(true);
-                }
-            } else {
-                onFailure.accept(e);
-            }
-        });
-        if (retrySecondTime.get()) {
-            if (log.isDebugEnabled()) {
-                log.debug("No candidate node found, likely all were upgraded in the meantime. Re-trying the original request.");
-            }
-            queryRunner.accept(onFailure);
-        }
-    }
-}
diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java
index b501967524a6b..29c471296b5d1 100644
--- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java
+++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java
@@ -218,7 +218,7 @@ public void testMultipleOnlyKnn() {
                 .addFetchField("text0")
                 .setSize(19),
             response -> {
-                assertEquals(51, response.getHits().getTotalHits().value);
+                assertEquals(51, response.getHits().getTotalHits().value());
                 assertEquals(19, response.getHits().getHits().length);
 
                 SearchHit hit = response.getHits().getAt(0);
@@ -355,7 +355,7 @@ public void testBM25AndKnnWithBucketAggregation() {
                 .setSize(11)
                 .addAggregation(AggregationBuilders.terms("sums").field("int")),
             response -> {
-                assertEquals(101, response.getHits().getTotalHits().value);
+                assertEquals(101, response.getHits().getTotalHits().value());
                 assertEquals(11, response.getHits().getHits().length);
 
                 SearchHit hit = response.getHits().getAt(0);
@@ -483,7 +483,7 @@ public void testBM25AndMultipleKnnWithAggregation() {
                 .addAggregation(AggregationBuilders.terms("sums").field("int"))
                 .setStats("search"),
             response -> {
-                assertEquals(51, response.getHits().getTotalHits().value);
+                assertEquals(51, response.getHits().getTotalHits().value());
                 assertEquals(19, response.getHits().getHits().length);
 
                 SearchHit hit = response.getHits().getAt(0);
diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java
index 7269d9c3e5e7f..ed26aa50ffa62 100644
--- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java
+++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java
@@ -217,7 +217,7 @@ public void testMultipleOnlyKnn() {
                 .addFetchField("text0")
                 .setSize(19),
             response -> {
-                assertEquals(51, response.getHits().getTotalHits().value);
+                assertEquals(51, response.getHits().getTotalHits().value());
                 assertEquals(19, response.getHits().getHits().length);
 
                 SearchHit hit = response.getHits().getAt(0);
@@ -356,7 +356,7 @@ public void testBM25AndKnnWithBucketAggregation() {
                 .setSize(11)
                 .addAggregation(AggregationBuilders.terms("sums").field("int")),
             response -> {
-                assertEquals(101, response.getHits().getTotalHits().value);
+                assertEquals(101, response.getHits().getTotalHits().value());
                 assertEquals(11, response.getHits().getHits().length);
 
                 SearchHit hit = response.getHits().getAt(0);
@@ -486,7 +486,7 @@ public void testBM25AndMultipleKnnWithAggregation() {
                 .addAggregation(AggregationBuilders.terms("sums").field("int"))
                 .setStats("search"),
             response -> {
-                assertEquals(51, response.getHits().getTotalHits().value);
+                assertEquals(51, response.getHits().getTotalHits().value());
                 assertEquals(19, response.getHits().getHits().length);
 
                 SearchHit hit = response.getHits().getAt(0);
diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java
index def26a613775a..37e1807d138aa 100644
--- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java
+++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java
@@ -8,6 +8,8 @@
 package org.elasticsearch.xpack.rank.rrf;
 
 import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchRequestBuilder;
@@ -18,6 +20,7 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -47,7 +50,6 @@
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -174,7 +176,10 @@ public void testRRFPagination() {
                 );
                 // this one retrieves docs 2 and 6 due to prefilter
                 StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-                    QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+                    QueryBuilders.boolQuery()
+                        .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                        .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                        .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
                 );
                 standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
                 // this one retrieves docs 2, 3, 6, and 7
@@ -195,8 +200,8 @@ public void testRRFPagination() {
                 ElasticsearchAssertions.assertResponse(req, resp -> {
                     assertNull(resp.pointInTimeId());
                     assertNotNull(resp.getHits().getTotalHits());
-                    assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
-                    assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+                    assertThat(resp.getHits().getTotalHits().value(), equalTo(6L));
+                    assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
                     assertThat(resp.getHits().getHits().length, lessThanOrEqualTo(size));
                     for (int k = 0; k < Math.min(size, resp.getHits().getHits().length); k++) {
                         assertThat(resp.getHits().getAt(k).getId(), equalTo(expectedDocIds.get(k + fDocs_to_fetch)));
@@ -221,7 +226,10 @@ public void testRRFWithAggs() {
         );
         // this one retrieves docs 2 and 6 due to prefilter
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         // this one retrieves docs 2, 3, 6, and 7
@@ -243,8 +251,8 @@ public void testRRFWithAggs() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getHits().length, equalTo(1));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
 
@@ -273,7 +281,10 @@ public void testRRFWithCollapse() {
         );
         // this one retrieves docs 2 and 6 due to prefilter
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         // this one retrieves docs 2, 3, 6, and 7
@@ -299,8 +310,8 @@ public void testRRFWithCollapse() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getHits().length, equalTo(4));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6"));
@@ -327,7 +338,10 @@ public void testRRFRetrieverWithCollapseAndAggs() {
         );
         // this one retrieves docs 2 and 6 due to prefilter
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         // this one retrieves docs 2, 3, 6, and 7
@@ -354,8 +368,8 @@ public void testRRFRetrieverWithCollapseAndAggs() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getHits().length, equalTo(4));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6"));
@@ -390,7 +404,10 @@ public void testMultipleRRFRetrievers() {
         );
         // this one retrieves docs 2 and 6 due to prefilter
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         // this one retrieves docs 2, 3, 6, and 7
@@ -426,8 +443,8 @@ public void testMultipleRRFRetrievers() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_7"));
             assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_6"));
@@ -453,7 +470,10 @@ public void testRRFExplainWithNamedRetrievers() {
         standard0.retrieverName("my_custom_retriever");
         // this one retrieves docs 2 and 6 due to prefilter
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         // this one retrieves docs 2, 3, 6, and 7
@@ -475,8 +495,8 @@ public void testRRFExplainWithNamedRetrievers() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getHits().length, equalTo(1));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(0).getExplanation().isMatch(), equalTo(true));
@@ -509,7 +529,10 @@ public void testRRFExplainWithAnotherNestedRRF() {
         standard0.retrieverName("my_custom_retriever");
         // this one retrieves docs 2 and 6 due to prefilter
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         // this one retrieves docs 2, 3, 6, and 7
@@ -543,8 +566,8 @@ public void testRRFExplainWithAnotherNestedRRF() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getHits().length, equalTo(1));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6"));
             assertThat(resp.getHits().getAt(0).getExplanation().isMatch(), equalTo(true));
@@ -568,16 +591,19 @@ public void testRRFExplainWithAnotherNestedRRF() {
         });
     }
 
-    public void testRRFInnerRetrieverSearchError() {
+    public void testRRFInnerRetrieverAll4xxSearchErrors() {
         final int rankWindowSize = 100;
         final int rankConstant = 10;
         SearchSourceBuilder source = new SearchSourceBuilder();
-        // this will throw an error during evaluation
+        // this will throw a 4xx error during evaluation
         StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(
             QueryBuilders.constantScoreQuery(QueryBuilders.rangeQuery(VECTOR_FIELD).gte(10))
         );
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         source.retriever(
@@ -591,10 +617,57 @@ public void testRRFInnerRetrieverSearchError() {
             )
         );
         SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
-        Exception ex = expectThrows(IllegalStateException.class, req::get);
-        assertThat(ex, instanceOf(IllegalStateException.class));
-        assertThat(ex.getMessage(), containsString("Search failed - some nested retrievers returned errors"));
-        assertThat(ex.getSuppressed().length, greaterThan(0));
+        Exception ex = expectThrows(ElasticsearchStatusException.class, req::get);
+        assertThat(ex, instanceOf(ElasticsearchStatusException.class));
+        assertThat(
+            ex.getMessage(),
+            containsString(
+                "[rrf] search failed - retrievers '[standard]' returned errors. All failures are attached as suppressed exceptions."
+            )
+        );
+        assertThat(ExceptionsHelper.status(ex), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(ex.getSuppressed().length, equalTo(1));
+        assertThat(ex.getSuppressed()[0].getCause().getCause(), instanceOf(IllegalArgumentException.class));
+    }
+
+    public void testRRFInnerRetrieverMultipleErrorsOne5xx() {
+        final int rankWindowSize = 100;
+        final int rankConstant = 10;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        // this will throw a 4xx error during evaluation
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(
+            QueryBuilders.constantScoreQuery(QueryBuilders.rangeQuery(VECTOR_FIELD).gte(10))
+        );
+        // this will throw a 5xx error
+        TestRetrieverBuilder testRetrieverBuilder = new TestRetrieverBuilder("val") {
+            @Override
+            public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+                searchSourceBuilder.aggregation(AggregationBuilders.avg("some_invalid_param"));
+            }
+        };
+        source.retriever(
+            new RRFRetrieverBuilder(
+                Arrays.asList(
+                    new CompoundRetrieverBuilder.RetrieverSource(standard0, null),
+                    new CompoundRetrieverBuilder.RetrieverSource(testRetrieverBuilder, null)
+                ),
+                rankWindowSize,
+                rankConstant
+            )
+        );
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        Exception ex = expectThrows(ElasticsearchStatusException.class, req::get);
+        assertThat(ex, instanceOf(ElasticsearchStatusException.class));
+        assertThat(
+            ex.getMessage(),
+            containsString(
+                "[rrf] search failed - retrievers '[standard, test]' returned errors. All failures are attached as suppressed exceptions."
+            )
+        );
+        assertThat(ExceptionsHelper.status(ex), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+        assertThat(ex.getSuppressed().length, equalTo(2));
+        assertThat(ex.getSuppressed()[0].getCause().getCause(), instanceOf(IllegalArgumentException.class));
+        assertThat(ex.getSuppressed()[1].getCause().getCause(), instanceOf(IllegalStateException.class));
     }
 
     public void testRRFInnerRetrieverErrorWhenExtractingToSource() {
@@ -613,7 +686,10 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder
             }
         };
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         source.retriever(
@@ -646,7 +722,10 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder
             }
         };
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         source.retriever(
@@ -703,14 +782,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         );
         assertResponse(
             client().prepareSearch(INDEX).setSource(new SearchSourceBuilder().retriever(rrf)),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, is(4L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), is(4L))
         );
         assertThat(numAsyncCalls.get(), equalTo(2));
 
         // check that we use the rewritten vector to build the explain query
         assertResponse(
             client().prepareSearch(INDEX).setSource(new SearchSourceBuilder().retriever(rrf).explain(true)),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, is(4L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), is(4L))
         );
         assertThat(numAsyncCalls.get(), equalTo(4));
     }
diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java
index ea251917cfae2..b1358f11bf633 100644
--- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java
+++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java
@@ -142,7 +142,10 @@ public void testRRFRetrieverWithNestedQuery() {
         );
         // this one retrieves docs 2 and 6 due to prefilter
         StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(
-            QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L)
+            QueryBuilders.boolQuery()
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L))
+                .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L))
         );
         standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
         // this one retrieves docs 6
@@ -164,8 +167,8 @@ public void testRRFRetrieverWithNestedQuery() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(3L));
-            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getTotalHits().value(), equalTo(3L));
+            assertThat(resp.getHits().getTotalHits().relation(), equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6"));
             assertThat((double) resp.getHits().getAt(0).getScore(), closeTo(0.1742, 1e-4));
             assertThat(
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
index 10aff2f4d68cd..fb20f834937d3 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
@@ -36,16 +36,17 @@
 
 /**
  * The builder to support RRF. Adds user-defined parameters for window size and rank constant.
+ *
+ * @deprecated RRF support is provided through the retriever framework. Please use {@link RRFRetrieverBuilder instead}
  */
+@Deprecated
 public class RRFRankBuilder extends RankBuilder {
 
-    public static final int DEFAULT_RANK_CONSTANT = 60;
-
     public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant");
 
     static final ConstructingObjectParser<RRFRankBuilder, Void> PARSER = new ConstructingObjectParser<>(RRFRankPlugin.NAME, args -> {
         int windowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0];
-        int rankConstant = args[1] == null ? DEFAULT_RANK_CONSTANT : (int) args[1];
+        int rankConstant = args[1] == null ? RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT : (int) args[1];
         return new RRFRankBuilder(windowSize, rankConstant);
     });
 
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
index 272df248e53e9..4cd10801b298c 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
@@ -19,7 +19,7 @@
 import java.util.Arrays;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.rank.rrf.RRFRankBuilder.DEFAULT_RANK_CONSTANT;
+import static org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT;
 
 /**
  * {@code RRFRankDoc} supports additional ranking information
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
index 12c43a2f169f8..792ff4eac3893 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.search.rank.RankBuilder;
 import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.retriever.CompoundRetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
@@ -31,7 +32,6 @@
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-import static org.elasticsearch.xpack.rank.rrf.RRFRankPlugin.NAME;
 
 /**
  * An rrf retriever is used to represent an rrf rank element, but
@@ -50,6 +50,7 @@ public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder<RRFRetri
     public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
     public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant");
 
+    public static final int DEFAULT_RANK_CONSTANT = 60;
     @SuppressWarnings("unchecked")
     static final ConstructingObjectParser<RRFRetrieverBuilder, RetrieverParserContext> PARSER = new ConstructingObjectParser<>(
         NAME,
@@ -57,8 +58,8 @@ public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder<RRFRetri
         args -> {
             List<RetrieverBuilder> childRetrievers = (List<RetrieverBuilder>) args[0];
             List<RetrieverSource> innerRetrievers = childRetrievers.stream().map(r -> new RetrieverSource(r, null)).toList();
-            int rankWindowSize = args[1] == null ? RRFRankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[1];
-            int rankConstant = args[2] == null ? RRFRankBuilder.DEFAULT_RANK_CONSTANT : (int) args[2];
+            int rankWindowSize = args[1] == null ? RankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[1];
+            int rankConstant = args[2] == null ? DEFAULT_RANK_CONSTANT : (int) args[2];
             return new RRFRetrieverBuilder(innerRetrievers, rankWindowSize, rankConstant);
         }
     );
@@ -82,7 +83,7 @@ public static RRFRetrieverBuilder fromXContent(XContentParser parser, RetrieverP
             throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]");
         }
         if (context.clusterSupportsFeature(RRF_RETRIEVER_COMPOSITION_SUPPORTED) == false) {
-            throw new UnsupportedOperationException("[rrf] retriever composition feature is not supported by all nodes in the cluster");
+            throw new IllegalArgumentException("[rrf] retriever composition feature is not supported by all nodes in the cluster");
         }
         if (RRFRankPlugin.RANK_RRF_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) {
             throw LicenseUtils.newComplianceException("Reciprocal Rank Fusion (RRF)");
diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java
index d324effe41c22..cae758457a2ac 100644
--- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java
+++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java
@@ -41,7 +41,7 @@ public static RRFRetrieverBuilder createRandomRRFRetrieverBuilder() {
         if (randomBoolean()) {
             rankWindowSize = randomIntBetween(1, 10000);
         }
-        int rankConstant = RRFRankBuilder.DEFAULT_RANK_CONSTANT;
+        int rankConstant = RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT;
         if (randomBoolean()) {
             rankConstant = randomIntBetween(1, 1000000);
         }
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
index 647540644ce9e..a5c346b386999 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
@@ -2,6 +2,8 @@ setup:
   - requires:
       cluster_features: "gte_v8.8.0"
       reason: 'rank added in 8.8'
+  - skip:
+      features: "warnings"
 
   - do:
       indices.create:
@@ -59,7 +61,14 @@ setup:
 ---
 "Simple rank with bm25 search and kNN search":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -94,7 +103,15 @@ setup:
 ---
 "Simple rank with multiple bm25 sub searches":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -135,7 +152,15 @@ setup:
 ---
 "Simple rank with multiple bm25 sub_searches and a knn search":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml
index b4893bfec0849..94a1457a7acc8 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml
@@ -63,7 +63,15 @@ setup:
 ---
 "Standard pagination within rank_window_size":
   # this test retrieves the same results from two queries, and applies a simple pagination skipping the first result
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -170,7 +178,15 @@ setup:
 ---
 "Standard pagination outside rank_window_size":
   # in this example, from starts *after* rank_window_size so, we expect 0 results to be returned
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -274,7 +290,15 @@ setup:
 ---
 "Standard pagination partially outside rank_window_size":
   # in this example we have that from starts *within* rank_window_size, but "from + size" goes over
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -384,7 +408,15 @@ setup:
   # queryA has a result set of [1, 2, 3, 4] and
   # queryB has a result set of [4, 3, 1, 2]
   # so for rank_constant=10, the expected order is [1, 4, 3, 2]
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -488,6 +520,9 @@ setup:
   - match: { hits.hits.1._id: "4" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -597,7 +632,15 @@ setup:
   # queryA has a result set of [5, 1] and
   # queryB has a result set of [4, 3, 1, 2]
   # so for rank_constant=10, the expected order is [1, 5, 4, 3, 2]
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -685,6 +728,9 @@ setup:
   - match: { hits.hits.1._id: "5" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -772,6 +818,9 @@ setup:
   - match: { hits.hits.1._id: "3" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -867,7 +916,15 @@ setup:
   # queryB has a result set of [4, 3]
   # so for rank_constant=10, the expected order is [5, 4, 1, 3],
   # and the rank_window_size-sized result set that we'd paginate over is [5, 4]
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -955,6 +1012,10 @@ setup:
   - match: { hits.hits.1._id: "4" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
+
       search:
         index: test
         body:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml
index bca39dea4ae57..36e70581f39f2 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml
@@ -67,7 +67,14 @@ setup:
 ---
 "RRF using single knn and single BM25 with a scripted metric aggregation":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -140,7 +147,14 @@ setup:
 ---
 "RRF using multi-knn only with a scripted metric aggregation":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -195,7 +209,14 @@ setup:
 ---
 "RRF using multi-knn and single BM25 with a scripted metric aggregation":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml
index 5718cd3455526..1b74ffee62a11 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml
@@ -75,7 +75,14 @@ setup:
 ---
 "using a top level knn and query":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -129,7 +136,15 @@ setup:
 ---
 "using sub_searches":
 
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -194,7 +209,14 @@ setup:
 ---
 "using named top level knn and query":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -251,7 +273,15 @@ setup:
 ---
 "using named sub_searches":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -320,7 +350,15 @@ setup:
 ---
 "using a mix of named and unnamed queries":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml
index e34885419c7f7..a9ddb4f902929 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml
@@ -172,7 +172,14 @@ setup:
 ---
 "using query and dfs knn search":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml
index 517c162c33e95..f3914843b80ec 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml
@@ -11,6 +11,8 @@ setup:
         body:
           mappings:
             properties:
+              id:
+                type: integer
               text:
                 type: text
               text_to_highlight:
@@ -39,6 +41,7 @@ setup:
         index: test
         id: "1"
         body:
+          id: 1
           text: "term1"
           vector: [1.0]
 
@@ -47,6 +50,7 @@ setup:
         index: test
         id: "2"
         body:
+          id: 2
           text: "term2"
           text_to_highlight: "search for the truth"
           keyword: "biology"
@@ -57,6 +61,7 @@ setup:
         index: test
         id: "3"
         body:
+          id: 3
           text: "term3"
           text_to_highlight: "nothing related"
           keyword: "technology"
@@ -67,6 +72,7 @@ setup:
         index: test
         id: "4"
         body:
+          id: 4
           text: "term4"
           vector: [4.0]
   - do:
@@ -74,6 +80,7 @@ setup:
         index: test
         id: "5"
         body:
+          id: 5
           text: "term5"
           text_to_highlight: "You know, for Search!"
           keyword: "technology"
@@ -81,9 +88,10 @@ setup:
           vector: [5.0]
   - do:
       index:
+        id: 6
         index: test
-        id: "6"
         body:
+          id: 6
           text: "term6"
           keyword: "biology"
           integer: 6
@@ -93,6 +101,7 @@ setup:
         index: test
         id: "7"
         body:
+          id: 7
           text: "term7"
           keyword: "astronomy"
           vector: [77.0]
@@ -102,6 +111,7 @@ setup:
         index: test
         id: "8"
         body:
+          id: 8
           text: "term8"
           keyword: "technology"
           nested: { views: 100}
@@ -110,6 +120,7 @@ setup:
         index: test
         id: "9"
         body:
+          id: 9
           text: "term9"
           integer: 2
           keyword: "technology"
@@ -439,7 +450,19 @@ setup:
               rank_window_size: 5
               rank_constant: 10
           size: 3
-          collapse: { field: keyword, inner_hits: { name: sub_hits, size: 2 } }
+          collapse: {
+            field: keyword,
+            inner_hits: {
+              name: sub_hits,
+              size: 2,
+              sort:
+                {
+                  id: {
+                    order: desc
+                  }
+                }
+            }
+          }
 
   - match: { hits.total : 9 }
 
@@ -456,8 +479,8 @@ setup:
 
   - match: { hits.hits.1.inner_hits.sub_hits.hits.total : 4 }
   - length: { hits.hits.1.inner_hits.sub_hits.hits.hits : 2 }
-  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "5" }
-  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "3" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "9" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "8" }
 
   - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
   - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "6" }
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml
index 3e758ae11f7e6..d9db1fe387625 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml
@@ -1,4 +1,9 @@
 setup:
+  - skip:
+      features:
+        - close_to
+        - contains
+
   - requires:
       cluster_features: ['rrf_retriever_composition_supported', 'text_similarity_reranker_retriever_supported']
       reason: need to have support for rrf and semantic reranking composition
@@ -149,8 +154,8 @@ setup:
   - match: { hits.total.value: 3 }
   - length: { hits.hits: 2 }
 
-  - match: { hits.hits.0._id: "doc_1" }
-  - match: { hits.hits.1._id: "doc_3" }
+  - contains: { hits.hits: { _id: "doc_1" } }
+  - contains: { hits.hits: { _id: "doc_3" } }
 
   - match: { aggregations.topics.buckets.0.key: "science" }
   - match: { aggregations.topics.buckets.0.doc_count: 2 }
@@ -301,7 +306,7 @@ setup:
                           standard: {
                             query: {
                               term: {
-                                topic: "science"
+                                subtopic: "astronomy"
                               }
                             }
                           }
@@ -321,14 +326,13 @@ setup:
 
   - match: { hits.hits.0._id: "doc_2" }
   - match: { hits.hits.1._id: "doc_1" }
-  - match: { hits.hits.2._id: "doc_3" }
 
-  - close_to: { hits.hits.0._explanation.value: { value: 0.6666667, error: 0.000001 } }
-  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.6666667\\].*/" }
+  - close_to: { hits.hits.0._explanation.value: { value: 0.833333, error: 0.0001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
   - match: {hits.hits.0._explanation.details.0.value: 2}
   - match: {hits.hits.0._explanation.details.0.description: "/rrf.score:.\\[0.33333334\\].*/" }
   - match: {hits.hits.0._explanation.details.0.details.0.details.0.description: "/ConstantScore.*/" }
-  - match: {hits.hits.0._explanation.details.1.value: 2}
-  - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.33333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" }
   - match: {hits.hits.0._explanation.details.1.details.0.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" }
-  - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/weight.*science.*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/weight.*astronomy.*/" }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestDeleteRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestDeleteRollupJobAction.java
index 3662fc94e4c7c..e80c69ae84122 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestDeleteRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestDeleteRollupJobAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.rollup.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
@@ -26,7 +25,7 @@ public class RestDeleteRollupJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(DELETE, "/_rollup/job/{id}").replaces(DELETE, "/_xpack/rollup/job/{id}/", RestApiVersion.V_7).build());
+        return List.of(new Route(DELETE, "/_rollup/job/{id}"));
     }
 
     @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java
index 5f3d64587d8e6..d60a5307ebe05 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.rollup.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -25,7 +24,7 @@ public class RestGetRollupCapsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(GET, "/_rollup/data/{id}").replaces(GET, "/_xpack/rollup/data/{id}/", RestApiVersion.V_7).build());
+        return List.of(new Route(GET, "/_rollup/data/{id}"));
     }
 
     @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java
index e8714116dad22..1dab108ad7a3c 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -27,9 +26,7 @@ public class RestGetRollupIndexCapsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/{index}/_rollup/data").replaces(GET, "/{index}/_xpack/rollup/data", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/{index}/_rollup/data"));
     }
 
     @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupJobsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupJobsAction.java
index 1b19a17fa2a79..3308425c08232 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupJobsAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupJobsAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.rollup.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -25,7 +24,7 @@ public class RestGetRollupJobsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(GET, "/_rollup/job/{id}").replaces(GET, "/_xpack/rollup/job/{id}/", RestApiVersion.V_7).build());
+        return List.of(new Route(GET, "/_rollup/job/{id}"));
     }
 
     @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java
index e434da37b7585..7ca5489ff5ddc 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.rollup.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -23,7 +22,7 @@ public class RestPutRollupJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(PUT, "/_rollup/job/{id}").replaces(PUT, "/_xpack/rollup/job/{id}", RestApiVersion.V_7).build());
+        return List.of(new Route(PUT, "/_rollup/job/{id}"));
     }
 
     @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java
index 3c84e67dcd010..b22d70cc8d789 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStartRollupJobAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.rollup.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -23,9 +22,7 @@ public class RestStartRollupJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_rollup/job/{id}/_start").replaces(POST, "/_xpack/rollup/job/{id}/_start", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_rollup/job/{id}/_start"));
     }
 
     @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java
index a5de3b6b22ce0..540dafb841c2d 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestStopRollupJobAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.rollup.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -24,9 +23,7 @@ public class RestStopRollupJobAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_rollup/job/{id}/_stop").replaces(POST, "/_xpack/rollup/job/{id}/_stop", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_rollup/job/{id}/_stop"));
     }
 
     @Override
diff --git a/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java b/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
index 2a17a4a1152cf..8df4e3a8dbea5 100644
--- a/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
+++ b/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
@@ -120,7 +120,7 @@ private void assertPinnedPromotions(PinnedQueryBuilder pqb, LinkedHashSet<String
         assertResponse(
             prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSize(size).setFrom(from).setSearchType(DFS_QUERY_THEN_FETCH),
             response -> {
-                long numHits = response.getHits().getTotalHits().value;
+                long numHits = response.getHits().getTotalHits().value();
                 assertThat(numHits, lessThanOrEqualTo((long) numRelevantDocs + pins.size()));
 
                 // Check pins are sorted by increasing score, (unlike organic, there are no duplicate scores)
@@ -193,7 +193,7 @@ public void testExhaustiveScoring() throws Exception {
 
     private void assertExhaustiveScoring(PinnedQueryBuilder pqb) {
         assertResponse(prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSearchType(DFS_QUERY_THEN_FETCH), response -> {
-            long numHits = response.getHits().getTotalHits().value;
+            long numHits = response.getHits().getTotalHits().value();
             assertThat(numHits, equalTo(2L));
         });
     }
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreQuery.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreQuery.java
index d9e65c385c610..2370a3dee6d03 100644
--- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreQuery.java
+++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreQuery.java
@@ -79,12 +79,10 @@ public Query rewrite(IndexSearcher searcher) throws IOException {
      */
     protected static class CappedBulkScorer extends BulkScorer {
         final BulkScorer bulkScorer;
-        final Weight weight;
         final float maxScore;
 
-        public CappedBulkScorer(BulkScorer bulkScorer, Weight weight, float maxScore) {
+        public CappedBulkScorer(BulkScorer bulkScorer, float maxScore) {
             this.bulkScorer = bulkScorer;
-            this.weight = weight;
             this.maxScore = maxScore;
         }
 
@@ -125,15 +123,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
         final Weight innerWeight = searcher.createWeight(query, scoreMode, boost);
         if (scoreMode.needsScores()) {
             return new CappedScoreWeight(this, innerWeight, maxScore) {
-                @Override
-                public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
-                    final BulkScorer innerScorer = innerWeight.bulkScorer(context);
-                    if (innerScorer == null) {
-                        return null;
-                    }
-                    return new CappedBulkScorer(innerScorer, this, maxScore);
-                }
-
                 @Override
                 public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                     ScorerSupplier innerScorerSupplier = innerWeight.scorerSupplier(context);
@@ -152,7 +141,13 @@ public Scorer get(long leadCost) throws IOException {
                                     return innerScorer;
                                 }
                             }
-                            return new CappedScorer(innerWeight, innerScorer, maxScore);
+                            return new CappedScorer(innerScorer, maxScore);
+                        }
+
+                        @Override
+                        public BulkScorer bulkScorer() throws IOException {
+                            final BulkScorer innerScorer = innerScorerSupplier.bulkScorer();
+                            return new CappedBulkScorer(innerScorer, maxScore);
                         }
 
                         @Override
@@ -166,15 +161,6 @@ public long cost() {
                 public Matches matches(LeafReaderContext context, int doc) throws IOException {
                     return innerWeight.matches(context, doc);
                 }
-
-                @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
-                    ScorerSupplier scorerSupplier = scorerSupplier(context);
-                    if (scorerSupplier == null) {
-                        return null;
-                    }
-                    return scorerSupplier.get(Long.MAX_VALUE);
-                }
             };
         } else {
             return innerWeight;
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreWeight.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreWeight.java
index 6ad3b9ce4ef85..ccc90e8f671a6 100644
--- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreWeight.java
+++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScoreWeight.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 
@@ -36,8 +37,22 @@ public boolean isCacheable(LeafReaderContext ctx) {
     }
 
     @Override
-    public Scorer scorer(LeafReaderContext context) throws IOException {
-        return new CappedScorer(this, innerWeight.scorer(context), maxScore);
+    public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+        ScorerSupplier innerScorerSupplier = innerWeight.scorerSupplier(context);
+        if (innerScorerSupplier == null) {
+            return null;
+        }
+        return new ScorerSupplier() {
+            @Override
+            public Scorer get(long leadCost) throws IOException {
+                return new CappedScorer(innerScorerSupplier.get(leadCost), maxScore);
+            }
+
+            @Override
+            public long cost() {
+                return innerScorerSupplier.cost();
+            }
+        };
     }
 
     @Override
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScorer.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScorer.java
index 57b2b62b77f6d..67813588ba3be 100644
--- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScorer.java
+++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/CappedScorer.java
@@ -9,15 +9,14 @@
 
 import org.apache.lucene.search.FilterScorer;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
 
 import java.io.IOException;
 
 public class CappedScorer extends FilterScorer {
     private final float maxScore;
 
-    public CappedScorer(Weight weight, Scorer delegate, float maxScore) {
-        super(delegate, weight);
+    public CappedScorer(Scorer delegate, float maxScore) {
+        super(delegate);
         this.maxScore = maxScore;
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
index faf41e7e655a8..ed42d86bc8c49 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
@@ -20,14 +20,18 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.indices.IndicesService;
@@ -36,6 +40,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.NodeRoles;
 import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -51,6 +56,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING;
+import static org.elasticsearch.cluster.node.DiscoveryNode.getRolesFromSettings;
 import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
@@ -76,14 +82,24 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     @Override
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         final Settings initialSettings = super.nodeSettings(nodeOrdinal, otherSettings);
-        if (DiscoveryNode.canContainData(otherSettings)) {
+
+        if (DiscoveryNode.canContainData(otherSettings)
+            && getRolesFromSettings(otherSettings).stream()
+                .anyMatch(
+                    nr -> nr.roleName().equals(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
+                        || nr.roleName().equals(DiscoveryNodeRole.DATA_ROLE.roleName())
+                )) {
             return Settings.builder()
                 .put(initialSettings)
                 // Have a shared cache of reasonable size available on each node because tests randomize over frozen and cold allocation
                 .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofMb(randomLongBetween(1, 10)))
                 .build();
         } else {
-            return initialSettings;
+            return Settings.builder()
+                .put(initialSettings)
+                // Have a shared cache of reasonable size available on each node because tests randomize over frozen and cold allocation
+                .putNull(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey())
+                .build();
         }
     }
 
@@ -305,7 +321,7 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
                 assertThat(newSearchResponse.getSuccessfulShards(), equalTo(totalShards));
                 assertThat(newSearchResponse.getFailedShards(), equalTo(0));
                 assertThat(newSearchResponse.getTotalShards(), equalTo(totalShards));
-                assertThat(newSearchResponse.getHits().getTotalHits().value, equalTo((long) numDocsWithinRange));
+                assertThat(newSearchResponse.getHits().getTotalHits().value(), equalTo((long) numDocsWithinRange));
             });
 
             // test with SearchShardsAPI
@@ -655,7 +671,7 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped()
                 assertThat(searchResponse.getFailedShards(), equalTo(indexOutsideSearchRangeShardCount));
                 assertThat(searchResponse.getSkippedShards(), equalTo(searchableSnapshotShardCount));
                 assertThat(searchResponse.getTotalShards(), equalTo(totalShards));
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(0L));
             });
         }
 
@@ -736,7 +752,7 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped()
                 // a shard that's available in order to construct the search response
                 assertThat(newSearchResponse.getSkippedShards(), equalTo(totalShards - 1));
                 assertThat(newSearchResponse.getTotalShards(), equalTo(totalShards));
-                assertThat(newSearchResponse.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(newSearchResponse.getHits().getTotalHits().value(), equalTo(0L));
             });
         });
 
@@ -850,7 +866,7 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
             SearchResponse response = client().search(request).actionGet();
             logger.info(
                 "[TEST DEBUG INFO] Search hits: {} Successful shards: {}, failed shards: {}, skipped shards: {}, total shards: {}",
-                response.getHits().getTotalHits().value,
+                response.getHits().getTotalHits().value(),
                 response.getSuccessfulShards(),
                 response.getFailedShards(),
                 response.getSkippedShards(),
@@ -955,6 +971,129 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
         }
     }
 
+    public void testCanMatchSkipsPartiallyMountedIndicesWhenFrozenNodesUnavailable() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
+        final String dataNodeHoldingRegularIndex = internalCluster().startNode(
+            NodeRoles.onlyRole(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE)
+        );
+        final String dataNodeHoldingSearchableSnapshot = internalCluster().startNode(
+            NodeRoles.onlyRole(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)
+        );
+
+        final String indexToMountInFrozen = "frozen-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final int shardCount = randomIntBetween(2, 3);
+        createIndexWithTimestampAndEventIngested(indexToMountInFrozen, shardCount, Settings.EMPTY);
+        final int numDocsFrozenIndex = between(350, 1000);
+        indexRandomDocs(indexToMountInFrozen, numDocsFrozenIndex);
+
+        final String regularIndex = "regular-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        createIndexWithTimestampAndEventIngested(
+            regularIndex,
+            shardCount,
+            Settings.builder()
+                .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), dataNodeHoldingRegularIndex)
+                .build()
+        );
+        int numDocsRegularIndex = between(100, 1000);
+        indexDocumentsWithTimestampAndEventIngestedDates(regularIndex, numDocsRegularIndex, TIMESTAMP_TEMPLATE_WITHIN_RANGE);
+
+        final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        createRepository(repositoryName, "mock");
+
+        final SnapshotId snapshotId = createSnapshot(repositoryName, "snapshot-1", List.of(indexToMountInFrozen)).snapshotId();
+        assertAcked(indicesAdmin().prepareDelete(indexToMountInFrozen));
+
+        final String partiallyMountedIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+
+        final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
+            partiallyMountedIndex,
+            repositoryName,
+            snapshotId.getName(),
+            indexToMountInFrozen,
+            Settings.EMPTY,
+            Strings.EMPTY_ARRAY,
+            false,
+            MountSearchableSnapshotRequest.Storage.SHARED_CACHE
+        );
+        client().execute(MountSearchableSnapshotAction.INSTANCE, mountRequest).actionGet();
+
+        ensureGreen(regularIndex, partiallyMountedIndex);
+
+        // Stop the node holding the searchable snapshots, and since we defined
+        // the index allocation criteria to require the searchable snapshot
+        // index to be allocated in that node, the shards should remain unassigned
+        internalCluster().stopNode(dataNodeHoldingSearchableSnapshot);
+        final IndexMetadata partiallyMountedIndexMetadata = getIndexMetadata(partiallyMountedIndex);
+        waitUntilAllShardsAreUnassigned(partiallyMountedIndexMetadata.getIndex());
+
+        {
+            // term query
+            TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("_tier", "data_content");
+            List<String> indicesToSearch = List.of(regularIndex, partiallyMountedIndex);
+            SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
+                .source(new SearchSourceBuilder().query(termQueryBuilder));
+
+            assertResponse(client().search(request), searchResponse -> {
+                // as we excluded the frozen tier we shouldn't get any failures
+                assertThat(searchResponse.getFailedShards(), equalTo(0));
+                // we should be receiving all the hits from the index that's in the data_content tier
+                assertNotNull(searchResponse.getHits().getTotalHits());
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numDocsRegularIndex));
+            });
+        }
+
+        {
+            // termS query
+            TermsQueryBuilder termsQueryBuilder = QueryBuilders.termsQuery("_tier", "data_hot", "data_content");
+            List<String> indicesToSearch = List.of(regularIndex, partiallyMountedIndex);
+            SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
+                .source(new SearchSourceBuilder().query(termsQueryBuilder));
+
+            assertResponse(client().search(request), searchResponse -> {
+                // as we excluded the frozen tier we shouldn't get any failures
+                assertThat(searchResponse.getFailedShards(), equalTo(0));
+                // we should be receiving all the hits from the index that's in the data_content tier
+                assertNotNull(searchResponse.getHits().getTotalHits());
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numDocsRegularIndex));
+            });
+        }
+
+        {
+            // bool term query
+            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("_tier", "data_frozen"));
+            List<String> indicesToSearch = List.of(regularIndex, partiallyMountedIndex);
+            SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
+                .source(new SearchSourceBuilder().query(boolQueryBuilder));
+
+            assertResponse(client().search(request), searchResponse -> {
+                // as we excluded the frozen tier we shouldn't get any failures
+                assertThat(searchResponse.getFailedShards(), equalTo(0));
+                // we should be receiving all the hits from the index that's in the data_content tier
+                assertNotNull(searchResponse.getHits().getTotalHits());
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numDocsRegularIndex));
+            });
+        }
+
+        {
+            // bool prefix, wildcard
+            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery()
+                .mustNot(randomFrom(QueryBuilders.wildcardQuery("_tier", "dat*ozen"), QueryBuilders.prefixQuery("_tier", "data_fro")));
+            List<String> indicesToSearch = List.of(regularIndex, partiallyMountedIndex);
+            SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
+                .source(new SearchSourceBuilder().query(boolQueryBuilder));
+
+            assertResponse(client().search(request), searchResponse -> {
+                // as we excluded the frozen tier we shouldn't get any failures
+                assertThat(searchResponse.getFailedShards(), equalTo(0));
+                // we should be receiving all the hits from the index that's in the data_content tier
+                assertNotNull(searchResponse.getHits().getTotalHits());
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numDocsRegularIndex));
+            });
+        }
+    }
+
     private void createIndexWithTimestampAndEventIngested(String indexName, int numShards, Settings extraSettings) throws IOException {
         assertAcked(
             indicesAdmin().prepareCreate(indexName)
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRecoverFromSnapshotIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRecoverFromSnapshotIntegTests.java
index 7615723860cff..9888afdd16499 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRecoverFromSnapshotIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRecoverFromSnapshotIntegTests.java
@@ -79,7 +79,7 @@ public void testSearchableSnapshotRelocationDoNotUseSnapshotBasedRecoveries() th
 
             ensureGreen(restoredIndexName);
 
-            assertHitCount(prepareSearch(restoredIndexName).setTrackTotalHits(true), totalHits.value);
+            assertHitCount(prepareSearch(restoredIndexName).setTrackTotalHits(true), totalHits.value());
 
             mockLog.assertAllExpectationsMatched();
         }
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java
index a3da932398fb1..1e76477378da2 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java
@@ -66,7 +66,7 @@ public void testRepositoryUsedBySearchableSnapshotCanBeUpdatedButNotUnregistered
             Storage storage = randomFrom(Storage.values());
             String restoredIndexName = (storage == Storage.FULL_COPY ? "fully-mounted-" : "partially-mounted-") + indexName + '-' + i;
             mountSnapshot(repositoryName, snapshotName, indexName, restoredIndexName, Settings.EMPTY, storage);
-            assertHitCount(prepareSearch(restoredIndexName).setTrackTotalHits(true), totalHits.value);
+            assertHitCount(prepareSearch(restoredIndexName).setTrackTotalHits(true), totalHits.value());
             mountedIndices[i] = restoredIndexName;
         }
 
@@ -183,7 +183,7 @@ public void testMountIndexWithDifferentDeletionOfSnapshot() throws Exception {
                 ? equalTo(Boolean.toString(deleteSnapshot))
                 : nullValue()
         );
-        assertHitCount(prepareSearch(mounted).setTrackTotalHits(true), totalHits.value);
+        assertHitCount(prepareSearch(mounted).setTrackTotalHits(true), totalHits.value());
 
         final String mountedAgain = randomValueOtherThan(mounted, () -> randomAlphaOfLength(10).toLowerCase(Locale.ROOT));
         final SnapshotRestoreException exception = expectThrows(
@@ -208,7 +208,7 @@ public void testMountIndexWithDifferentDeletionOfSnapshot() throws Exception {
                 ? equalTo(Boolean.toString(deleteSnapshot))
                 : nullValue()
         );
-        assertHitCount(prepareSearch(mountedAgain).setTrackTotalHits(true), totalHits.value);
+        assertHitCount(prepareSearch(mountedAgain).setTrackTotalHits(true), totalHits.value());
 
         assertAcked(indicesAdmin().prepareDelete(mountedAgain));
         assertAcked(indicesAdmin().prepareDelete(mounted));
@@ -240,7 +240,7 @@ public void testDeletionOfSnapshotSettingCannotBeUpdated() throws Exception {
                 ? equalTo(Boolean.toString(deleteSnapshot))
                 : nullValue()
         );
-        assertHitCount(prepareSearch(mounted).setTrackTotalHits(true), totalHits.value);
+        assertHitCount(prepareSearch(mounted).setTrackTotalHits(true), totalHits.value());
 
         if (randomBoolean()) {
             assertAcked(indicesAdmin().prepareClose(mounted));
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java
index 40b7e08936fa3..7eaf5d8f060c6 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java
@@ -179,7 +179,7 @@ public void testCleanUpAfterIndicesAreDeleted() throws Exception {
                             )
                             .setSize(0),
                         res -> {
-                            final long remainingEntriesInCache = res.getHits().getTotalHits().value;
+                            final long remainingEntriesInCache = res.getHits().getTotalHits().value();
                             if (indicesToDelete.contains(mountedIndex)) {
                                 assertThat(remainingEntriesInCache, equalTo(0L));
                             } else if (snapshotId.equals(SNAPSHOT_SNAPSHOT_ID_SETTING.get(indexSettings))) {
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java
index a21e3e6beabce..21e67212f1f51 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java
@@ -549,7 +549,7 @@ public void run() {
                 try (listeners) {
                     executeSearch(new SearchRequest().source(getSearchSourceBuilder().trackTotalHits(true)), (searchResponse, refs) -> {
                         assert total.get() == 0L;
-                        total.set(searchResponse.getHits().getTotalHits().value);
+                        total.set(searchResponse.getHits().getTotalHits().value());
                         handleSearchResponse(searchResponse, refs);
                     });
                 }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java
index da08c6b38819b..a7fb5571995b3 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -173,9 +174,10 @@ long getCacheSize(ShardId shardId, SnapshotId snapshotId, Predicate<Path> predic
                         final Bits liveDocs = leafReaderContext.reader().getLiveDocs();
                         final IntPredicate isLiveDoc = liveDocs == null ? i -> true : liveDocs::get;
                         final DocIdSetIterator docIdSetIterator = scorer.iterator();
+                        StoredFields storedFields = leafReaderContext.reader().storedFields();
                         while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
                             if (isLiveDoc.test(docIdSetIterator.docID())) {
-                                final Document document = leafReaderContext.reader().document(docIdSetIterator.docID());
+                                final Document document = storedFields.document(docIdSetIterator.docID());
                                 final String cacheFileId = getValue(document, CACHE_ID_FIELD);
                                 if (predicate.test(snapshotCacheDir.resolve(cacheFileId))) {
                                     long size = buildCacheFileRanges(document).stream().mapToLong(ByteRange::length).sum();
@@ -423,9 +425,10 @@ static Map<String, Document> loadDocuments(Path directoryPath) throws IOExceptio
                 for (LeafReaderContext leafReaderContext : indexReader.leaves()) {
                     final LeafReader leafReader = leafReaderContext.reader();
                     final Bits liveDocs = leafReader.getLiveDocs();
+                    final StoredFields storedFields = leafReader.storedFields();
                     for (int i = 0; i < leafReader.maxDoc(); i++) {
                         if (liveDocs == null || liveDocs.get(i)) {
-                            final Document document = leafReader.document(i);
+                            final Document document = storedFields.document(i);
                             logger.trace("loading document [{}]", document);
                             documents.put(getValue(document, CACHE_ID_FIELD), document);
                         }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java
index 81cf205c13dd2..4711043fff281 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.ReadAdvice;
 import org.elasticsearch.blobcache.BlobCacheUtils;
 import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
@@ -35,7 +36,7 @@ public class CachedBlobContainerIndexInput extends MetadataCachingIndexInput {
      * a complete part of the {@link #fileInfo} at once in the cache and should not be
      * used for anything else than what the {@link #prefetchPart(int, Supplier)} method does.
      */
-    public static final IOContext CACHE_WARMING_CONTEXT = new IOContext();
+    public static final IOContext CACHE_WARMING_CONTEXT = new IOContext(IOContext.Context.DEFAULT, null, null, ReadAdvice.NORMAL);
 
     private static final Logger logger = LogManager.getLogger(CachedBlobContainerIndexInput.class);
 
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
index 41121453e41a4..4ee2bf7e65633 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
@@ -348,8 +348,8 @@ public static Tuple<String, byte[]> randomChecksumBytes(byte[] bytes) throws IOE
      * uses a different buffer size for them.
      */
     public static IOContext randomIOContext() {
-        final IOContext ioContext = randomFrom(IOContext.DEFAULT, IOContext.READ, IOContext.READONCE);
-        assert ioContext.context != IOContext.Context.MERGE;
+        final IOContext ioContext = randomFrom(IOContext.DEFAULT, IOContext.READONCE);
+        assert ioContext.context() != IOContext.Context.MERGE;
         return ioContext;
     }
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectoryTests.java
index c97d6cb4cab08..eab6f1a629f36 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectoryTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectoryTests.java
@@ -179,7 +179,7 @@ public void testSupportsNoOpCommits() throws IOException {
             final TopDocs topDocs = newSearcher(directoryReader).search(new MatchAllDocsQuery(), 1);
             assertThat(topDocs.totalHits, equalTo(new TotalHits(1L, TotalHits.Relation.EQUAL_TO)));
             assertThat(topDocs.scoreDocs.length, equalTo(1));
-            assertThat(directoryReader.document(topDocs.scoreDocs[0].doc).getField("foo").stringValue(), equalTo("bar"));
+            assertThat(directoryReader.storedFields().document(topDocs.scoreDocs[0].doc).getField("foo").stringValue(), equalTo("bar"));
         }
 
         try (IndexWriter indexWriter = new IndexWriter(inMemoryNoOpCommitDirectory, new IndexWriterConfig())) {
@@ -226,7 +226,7 @@ public void testSupportsDeletes() throws IOException {
             final TopDocs topDocs = newSearcher(directoryReader).search(new MatchAllDocsQuery(), 1);
             assertThat(topDocs.totalHits, equalTo(new TotalHits(1L, TotalHits.Relation.EQUAL_TO)));
             assertThat(topDocs.scoreDocs.length, equalTo(1));
-            assertThat(directoryReader.document(topDocs.scoreDocs[0].doc).getField("foo").stringValue(), equalTo("bar"));
+            assertThat(directoryReader.storedFields().document(topDocs.scoreDocs[0].doc).getField("foo").stringValue(), equalTo("bar"));
         }
 
         assertEquals(1, DirectoryReader.listCommits(inMemoryNoOpCommitDirectory).size());
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
index e65c4a60f89d5..98df96eca7772 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
@@ -401,9 +401,9 @@ public void testChecksumBlobContainerIndexInput() throws Exception {
             false, // no prewarming in this test because we want to ensure that files are accessed on purpose
             (directory, snapshotDirectory) -> {
                 for (String fileName : randomSubsetOf(Arrays.asList(snapshotDirectory.listAll()))) {
-                    final long checksum;
-                    try (IndexInput input = directory.openInput(fileName, Store.READONCE_CHECKSUM)) {
-                        checksum = CodecUtil.checksumEntireFile(input);
+                    final long expectedChecksum;
+                    try (IndexInput input = directory.openInput(fileName, IOContext.READONCE)) {
+                        expectedChecksum = CodecUtil.checksumEntireFile(input);
                     }
 
                     final long snapshotChecksum;
@@ -418,9 +418,9 @@ public void testChecksumBlobContainerIndexInput() throws Exception {
                     }
 
                     assertThat(
-                        "Expected checksum [" + checksum + "] but got [" + snapshotChecksum + ']',
+                        "Expected checksum [" + expectedChecksum + "] but got [" + snapshotChecksum + ']',
                         snapshotChecksum,
-                        equalTo(checksum)
+                        equalTo(expectedChecksum)
                     );
                     assertThat(
                         "File [" + fileName + "] should have been read from heap",
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityDlsAndFlsRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityDlsAndFlsRestIT.java
index 6ffa09dc1f265..6d9110b564862 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityDlsAndFlsRestIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityDlsAndFlsRestIT.java
@@ -224,7 +224,7 @@ protected void assertSearchResponseContainsEmptyResult(Response response) {
             assertOK(response);
             SearchResponse searchResponse = SearchResponseUtils.responseAsSearchResponse(response);
             try {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(0L));
             } finally {
                 searchResponse.decRef();
             }
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrIT.java
index 767452e6fcae7..4b994ce82d92f 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrIT.java
@@ -282,7 +282,7 @@ private void verifyReplicatedDocuments(long numberOfDocs, String... indices) thr
                 searchResponse = SearchResponseUtils.parseSearchResponse(parser);
             }
             try {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numberOfDocs));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(numberOfDocs));
                 assertThat(
                     Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).collect(Collectors.toUnmodifiableSet()),
                     equalTo(Set.of(indices))
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrMigrationIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrMigrationIT.java
index d5e77c1694640..1602a097b1b08 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrMigrationIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCcrMigrationIT.java
@@ -362,7 +362,7 @@ private void verifyReplicatedDocuments(long numberOfDocs, String... indices) thr
             assertOK(response);
             final SearchResponse searchResponse = SearchResponseUtils.parseSearchResponse(responseAsParser(response));
             try {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numberOfDocs));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(numberOfDocs));
                 assertThat(
                     Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).collect(Collectors.toUnmodifiableSet()),
                     equalTo(Set.of(indices))
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityMutualTlsIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityMutualTlsIT.java
index 8b18359fb8310..1345e275fab17 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityMutualTlsIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityMutualTlsIT.java
@@ -119,7 +119,7 @@ public void testCrossClusterSearch() throws Exception {
                 responseAsParser(performRequestWithRemoteMetricUser(metricSearchRequest))
             );
             try {
-                assertThat(metricSearchResponse.getHits().getTotalHits().value, equalTo(4L));
+                assertThat(metricSearchResponse.getHits().getTotalHits().value(), equalTo(4L));
                 assertThat(
                     Arrays.stream(metricSearchResponse.getHits().getHits()).map(SearchHit::getIndex).collect(Collectors.toSet()),
                     containsInAnyOrder("shared-metrics")
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java
index 69331fa448113..4cbd1cab21af9 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java
@@ -419,7 +419,7 @@ public void testCrossClusterSearch() throws Exception {
                 responseAsParser(performRequestWithRemoteMetricUser(metricSearchRequest))
             );
             try {
-                assertThat(metricSearchResponse.getHits().getTotalHits().value, equalTo(4L));
+                assertThat(metricSearchResponse.getHits().getTotalHits().value(), equalTo(4L));
                 assertThat(
                     Arrays.stream(metricSearchResponse.getHits().getHits()).map(SearchHit::getIndex).collect(Collectors.toSet()),
                     containsInAnyOrder("shared-metrics")
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySpecialUserIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySpecialUserIT.java
index 505b82b39b960..53c622898476a 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySpecialUserIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySpecialUserIT.java
@@ -230,7 +230,7 @@ public void testAnonymousUserFromQueryClusterWorks() throws Exception {
                     Arrays.stream(searchResponse5.getHits().getHits()).map(SearchHit::getIndex).collect(Collectors.toList()),
                     containsInAnyOrder(".security-7")
                 );
-                assertThat(searchResponse5.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+                assertThat(searchResponse5.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
             } finally {
                 searchResponse5.decRef();
             }
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityTopologyRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityTopologyRestIT.java
index 3871029b3b44b..6fa3ef1b4ef63 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityTopologyRestIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityTopologyRestIT.java
@@ -134,7 +134,7 @@ public void testCrossClusterScrollWithSniffModeWhenSomeRemoteNodesAreNotDirectly
             final Request scrollRequest = new Request("GET", "/_search/scroll");
             final String scrollId;
             try {
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(6L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(6L));
                 assertThat(Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).toList(), contains("shared-metrics"));
                 documentFieldValues.add(searchResponse.getHits().getHits()[0].getSourceAsMap().get("name"));
                 scrollId = searchResponse.getScrollId();
@@ -153,7 +153,7 @@ public void testCrossClusterScrollWithSniffModeWhenSomeRemoteNodesAreNotDirectly
                     responseAsParser(performRequestWithRemoteMetricUser(scrollRequest))
                 );
                 try {
-                    assertThat(scrollResponse.getHits().getTotalHits().value, equalTo(6L));
+                    assertThat(scrollResponse.getHits().getTotalHits().value(), equalTo(6L));
                     assertThat(
                         Arrays.stream(scrollResponse.getHits().getHits()).map(SearchHit::getIndex).toList(),
                         contains("shared-metrics")
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
index 853d0fd9318ae..4405ef575b24f 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
@@ -174,6 +174,7 @@ public class Constants {
         "cluster:admin/xpack/enrich/reindex",
         "cluster:admin/xpack/inference/delete",
         "cluster:admin/xpack/inference/put",
+        "cluster:admin/xpack/inference/update",
         "cluster:admin/xpack/license/basic_status",
         // "cluster:admin/xpack/license/delete",
         "cluster:admin/xpack/license/feature_usage",
@@ -244,6 +245,7 @@ public class Constants {
         "cluster:admin/xpack/query_rules/get",
         "cluster:admin/xpack/query_rules/list",
         "cluster:admin/xpack/query_rules/put",
+        "cluster:admin/xpack/query_rules/test",
         "cluster:admin/xpack/rollup/delete",
         "cluster:admin/xpack/rollup/put",
         "cluster:admin/xpack/rollup/start",
@@ -501,6 +503,9 @@ public class Constants {
         "indices:admin/data_stream/lifecycle/get",
         "indices:admin/data_stream/lifecycle/put",
         "indices:admin/data_stream/lifecycle/explain",
+        "indices:admin/data_stream/options/delete",
+        "indices:admin/data_stream/options/get",
+        "indices:admin/data_stream/options/put",
         "indices:admin/delete",
         "indices:admin/flush",
         "indices:admin/flush[s]",
diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
index 1abb9bbb067dc..523f04fb436f4 100644
--- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
+++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.TestSecurityClient;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -41,9 +42,7 @@
 public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase {
     private TestSecurityClient securityClient;
 
-    @ClassRule
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .nodes(2)
+    public static LocalClusterConfigProvider commonTrialSecurityClusterConfig = cluster -> cluster.nodes(2)
         .distribution(DistributionType.DEFAULT)
         .setting("xpack.ml.enabled", "false")
         .setting("xpack.license.self_generated.type", "trial")
@@ -62,8 +61,10 @@ public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase
         .user("admin_user", "admin-password", ROOT_USER_ROLE, true)
         .user("security_test_user", "security-test-password", "security_test_role", false)
         .user("x_pack_rest_user", "x-pack-test-password", ROOT_USER_ROLE, true)
-        .user("cat_test_user", "cat-test-password", "cat_test_role", false)
-        .build();
+        .user("cat_test_user", "cat-test-password", "cat_test_role", false);
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().apply(commonTrialSecurityClusterConfig).build();
 
     @Override
     protected String getTestRestCluster() {
diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/rolemapping/RoleMappingRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/rolemapping/RoleMappingRestIT.java
new file mode 100644
index 0000000000000..51970af4b88a0
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/rolemapping/RoleMappingRestIT.java
@@ -0,0 +1,268 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.rolemapping;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
+import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RoleMappingRestIT extends ESRestTestCase {
+    private static final String settingsJson = """
+        {
+             "metadata": {
+                 "version": "1",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "role-mapping-1": {
+                          "enabled": true,
+                          "roles": [ "role_1" ],
+                          "rules": { "field": { "username": "no_user" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                             "_foo": "something",
+                             "_es_reserved_role_mapping_name": "ignored"
+                          }
+                       },
+                       "role-mapping-2": {
+                          "enabled": true,
+                          "roles": [ "role_2" ],
+                          "rules": { "field": { "username": "no_user" } }
+                       },
+                       "role-mapping-3": {
+                          "enabled": true,
+                          "roles": [ "role_3" ],
+                          "rules": { "field": { "username": "no_user" } },
+                          "metadata": {
+                             "_read_only" : { "field": 1 },
+                             "_es_reserved_role_mapping_name": { "still_ignored": true }
+                          }
+                       }
+                 }
+             }
+        }""";
+    private static final ExpressionRoleMapping clusterStateMapping1 = new ExpressionRoleMapping(
+        "role-mapping-1-read-only-operator-mapping",
+        new FieldExpression("username", List.of(new FieldExpression.FieldValue("no_user"))),
+        List.of("role_1"),
+        null,
+        Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something", "_read_only", true),
+        true
+    );
+    private static final ExpressionRoleMapping clusterStateMapping2 = new ExpressionRoleMapping(
+        "role-mapping-2-read-only-operator-mapping",
+        new FieldExpression("username", List.of(new FieldExpression.FieldValue("no_user"))),
+        List.of("role_2"),
+        null,
+        Map.of("_read_only", true),
+        true
+    );
+    private static final ExpressionRoleMapping clusterStateMapping3 = new ExpressionRoleMapping(
+        "role-mapping-3-read-only-operator-mapping",
+        new FieldExpression("username", List.of(new FieldExpression.FieldValue("no_user"))),
+        List.of("role_3"),
+        null,
+        Map.of("_read_only", true),
+        true
+    );
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .apply(SecurityOnTrialLicenseRestTestCase.commonTrialSecurityClusterConfig)
+        .configFile("operator/settings.json", Resource.fromString(settingsJson))
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public void testGetRoleMappings() throws IOException {
+        expectMappings(List.of(clusterStateMapping1, clusterStateMapping2, clusterStateMapping3));
+        expectMappings(List.of(clusterStateMapping1), "role-mapping-1");
+        expectMappings(List.of(clusterStateMapping1, clusterStateMapping3), "role-mapping-1", "role-mapping-3");
+        expectMappings(List.of(clusterStateMapping1), clusterStateMapping1.getName());
+        expectMappings(List.of(clusterStateMapping1), clusterStateMapping1.getName(), "role-mapping-1");
+
+        expect404(() -> getMappings("role-mapping-4"));
+        expect404(() -> getMappings("role-mapping-4-read-only-operator-mapping"));
+
+        ExpressionRoleMapping nativeMapping1 = expressionRoleMapping("role-mapping-1");
+        putMapping(nativeMapping1, createOrUpdateWarning(nativeMapping1.getName()));
+
+        ExpressionRoleMapping nativeMapping4 = expressionRoleMapping("role-mapping-4");
+        putMapping(nativeMapping4);
+
+        expectMappings(List.of(clusterStateMapping1, clusterStateMapping2, clusterStateMapping3, nativeMapping1, nativeMapping4));
+        expectMappings(List.of(clusterStateMapping1, nativeMapping1), "role-mapping-1");
+        expectMappings(List.of(clusterStateMapping1, nativeMapping1), "role-mapping-1", clusterStateMapping1.getName());
+        expectMappings(List.of(clusterStateMapping1), clusterStateMapping1.getName());
+        expectMappings(List.of(nativeMapping4), "role-mapping-4");
+        expectMappings(List.of(nativeMapping4), "role-mapping-4", "role-mapping-4-read-only-operator-mapping");
+    }
+
+    public void testPutAndDeleteRoleMappings() throws IOException {
+        {
+            var ex = expectThrows(
+                ResponseException.class,
+                () -> putMapping(expressionRoleMapping("role-mapping-1-read-only-operator-mapping"))
+            );
+            assertThat(
+                ex.getMessage(),
+                containsString(
+                    "Invalid mapping name [role-mapping-1-read-only-operator-mapping]. "
+                        + "[-read-only-operator-mapping] is not an allowed suffix"
+                )
+            );
+        }
+
+        // Also fails even if a CS role mapping with that name does not exist
+        {
+            var ex = expectThrows(
+                ResponseException.class,
+                () -> putMapping(expressionRoleMapping("role-mapping-4-read-only-operator-mapping"))
+            );
+            assertThat(
+                ex.getMessage(),
+                containsString(
+                    "Invalid mapping name [role-mapping-4-read-only-operator-mapping]. "
+                        + "[-read-only-operator-mapping] is not an allowed suffix"
+                )
+            );
+        }
+
+        assertOK(putMapping(expressionRoleMapping("role-mapping-1"), createOrUpdateWarning("role-mapping-1")));
+
+        assertOK(deleteMapping("role-mapping-1", deletionWarning("role-mapping-1")));
+
+        // 404 without warnings if no native mapping exists
+        expect404(() -> deleteMapping("role-mapping-1"));
+    }
+
+    private static void expect404(ThrowingRunnable clientCall) {
+        var ex = expectThrows(ResponseException.class, clientCall);
+        assertThat(ex.getResponse().getStatusLine().getStatusCode(), equalTo(404));
+    }
+
+    private static Response putMapping(ExpressionRoleMapping roleMapping) throws IOException {
+        return putMapping(roleMapping, null);
+    }
+
+    private static Response putMapping(ExpressionRoleMapping roleMapping, @Nullable String warning) throws IOException {
+        Request request = new Request("PUT", "/_security/role_mapping/" + roleMapping.getName());
+        XContentBuilder xContent = roleMapping.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS);
+        request.setJsonEntity(BytesReference.bytes(xContent).utf8ToString());
+        if (warning != null) {
+            request.setOptions(
+                RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> warnings.equals(List.of(warning)) == false).build()
+            );
+        }
+        return client().performRequest(request);
+    }
+
+    private static Response deleteMapping(String name) throws IOException {
+        return deleteMapping(name, null);
+    }
+
+    private static Response deleteMapping(String name, @Nullable String warning) throws IOException {
+        Request request = new Request("DELETE", "/_security/role_mapping/" + name);
+        if (warning != null) {
+            request.setOptions(
+                RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> warnings.equals(List.of(warning)) == false).build()
+            );
+        }
+        return client().performRequest(request);
+    }
+
+    private static ExpressionRoleMapping expressionRoleMapping(String name) {
+        return new ExpressionRoleMapping(
+            name,
+            new FieldExpression("username", List.of(new FieldExpression.FieldValue(randomAlphaOfLength(10)))),
+            List.of(randomAlphaOfLength(5)),
+            null,
+            Map.of(),
+            true
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void expectMappings(List<ExpressionRoleMapping> expectedMappings, String... requestedMappingNames) throws IOException {
+        Map<String, Object> map = responseAsMap(getMappings(requestedMappingNames));
+        assertThat(
+            map.keySet(),
+            containsInAnyOrder(expectedMappings.stream().map(ExpressionRoleMapping::getName).toList().toArray(new String[0]))
+        );
+        List<ExpressionRoleMapping> actualMappings = new ArrayList<>();
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            XContentParser body = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, (Map<String, ?>) entry.getValue());
+            ExpressionRoleMapping actual = ExpressionRoleMapping.parse(entry.getKey(), body);
+            actualMappings.add(actual);
+        }
+        assertThat(actualMappings, containsInAnyOrder(expectedMappings.toArray(new ExpressionRoleMapping[0])));
+    }
+
+    private static Response getMappings(String... requestedMappingNames) throws IOException {
+        return client().performRequest(new Request("GET", "/_security/role_mapping/" + String.join(",", requestedMappingNames)));
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    private static String createOrUpdateWarning(String mappingName) {
+        return "A read-only role mapping with the same name ["
+            + mappingName
+            + "] has been previously defined in a configuration file. "
+            + "Both role mappings will be used to determine role assignments.";
+    }
+
+    private static String deletionWarning(String mappingName) {
+        return "A read-only role mapping with the same name ["
+            + mappingName
+            + "] has previously been defined in a configuration file. "
+            + "The native role mapping was deleted, but the read-only mapping will remain active "
+            + "and will be used to determine role assignments.";
+    };
+}
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java
index b1a76a4559812..9a1d653132d2d 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java
@@ -81,7 +81,7 @@ public void testDateMathExpressionsCanBeAuthorized() throws Exception {
 
         assertResponse(
             client.prepareMultiSearch().add(client.prepareSearch(expression).setQuery(QueryBuilders.matchAllQuery()).request()),
-            multiSearchResponse -> assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L))
+            multiSearchResponse -> assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L))
         );
 
         UpdateResponse updateResponse = client.prepareUpdate(expression, response.getId())
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
index b0572b265a45b..a5f827c2a4b53 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
@@ -437,7 +437,7 @@ private void assertSearchResponse(SearchRequestBuilder requestBuilder, Set<Strin
         var searchResponse = requestBuilder.get();
         try {
             assertThat(searchResponse.getFailedShards(), equalTo(0));
-            assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) docIds.size()));
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) docIds.size()));
             final SearchHit[] hits = searchResponse.getHits().getHits();
             assertThat(Arrays.stream(hits).map(SearchHit::getId).collect(Collectors.toUnmodifiableSet()), equalTo(docIds));
             if (fieldNames != null) {
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java
index 1bf7d89347755..73aebd4fc7db3 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java
@@ -116,7 +116,7 @@ public void testDuelWithAliasFilters() throws Exception {
                 client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user" + roleI, USERS_PASSWD)))
                     .prepareSearch("test"),
                 searchResponse1 -> assertResponse(prepareSearch("alias" + role), searchResponse2 -> {
-                    assertThat(searchResponse1.getHits().getTotalHits().value, equalTo(searchResponse2.getHits().getTotalHits().value));
+                    assertThat(searchResponse1.getHits().getTotalHits().value(), equalTo(searchResponse2.getHits().getTotalHits().value()));
                     for (int hitI = 0; hitI < searchResponse1.getHits().getHits().length; hitI++) {
                         assertThat(searchResponse1.getHits().getAt(hitI).getId(), equalTo(searchResponse2.getHits().getAt(hitI).getId()));
                     }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
index c0866fa7ea694..87ca7d279c709 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
@@ -474,13 +474,13 @@ public void testMSearch() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("id"), is(1));
 
                     assertFalse(response.getResponses()[1].isFailure());
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("id"), is(1));
@@ -495,13 +495,13 @@ public void testMSearch() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("id"), is(2));
 
                     assertFalse(response.getResponses()[1].isFailure());
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("id"), is(2));
@@ -522,7 +522,7 @@ public void testMSearch() throws Exception {
                     ),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(2L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(2L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("id"), is(1));
@@ -531,7 +531,7 @@ public void testMSearch() throws Exception {
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(1).getSourceAsMap().get("id"), is(2));
 
                     assertFalse(response.getResponses()[1].isFailure());
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(2L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(2L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("id"), is(1));
@@ -898,7 +898,7 @@ public void testKnnSearch() throws Exception {
                 .addFetchField("field1")
                 .setSize(10),
             response -> {
-                assertEquals(5, response.getHits().getTotalHits().value);
+                assertEquals(5, response.getHits().getTotalHits().value());
                 assertEquals(5, response.getHits().getHits().length);
                 for (SearchHit hit : response.getHits().getHits()) {
                     assertNotNull(hit.field("field1"));
@@ -914,7 +914,7 @@ public void testKnnSearch() throws Exception {
                 .addFetchField("field2")
                 .setSize(10),
             response -> {
-                assertEquals(5, response.getHits().getTotalHits().value);
+                assertEquals(5, response.getHits().getTotalHits().value());
                 assertEquals(5, response.getHits().getHits().length);
                 for (SearchHit hit : response.getHits().getHits()) {
                     assertNotNull(hit.field("field2"));
@@ -929,7 +929,7 @@ public void testKnnSearch() throws Exception {
                 .setQuery(query)
                 .setSize(10),
             response -> {
-                assertEquals(10, response.getHits().getTotalHits().value);
+                assertEquals(10, response.getHits().getTotalHits().value());
                 assertEquals(10, response.getHits().getHits().length);
             }
         );
@@ -1265,7 +1265,7 @@ public void testScroll() throws Exception {
                 .get();
             do {
                 assertNoFailures(response);
-                assertThat(response.getHits().getTotalHits().value, is((long) numVisible));
+                assertThat(response.getHits().getTotalHits().value(), is((long) numVisible));
                 assertThat(response.getHits().getAt(0).getSourceAsMap().size(), is(1));
                 assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
 
@@ -1325,7 +1325,7 @@ public void testReaderId() throws Exception {
                     .setQuery(termQuery("field1", "value1"))
                     .get();
                 assertNoFailures(response);
-                assertThat(response.getHits().getTotalHits().value, is((long) numVisible));
+                assertThat(response.getHits().getTotalHits().value(), is((long) numVisible));
                 assertThat(response.getHits().getAt(0).getSourceAsMap().size(), is(1));
                 assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
             }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java
index 34eecd57b53d5..01020a428c318 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java
@@ -208,7 +208,7 @@ public void testDuel() throws Exception {
                 prepareSearch("test").addSort("id", SortOrder.ASC)
                     .setQuery(QueryBuilders.boolQuery().should(QueryBuilders.termQuery("field1", "value"))),
                 expected -> {
-                    assertThat(actual.getHits().getTotalHits().value, equalTo(expected.getHits().getTotalHits().value));
+                    assertThat(actual.getHits().getTotalHits().value(), equalTo(expected.getHits().getTotalHits().value()));
                     assertThat(actual.getHits().getHits().length, equalTo(expected.getHits().getHits().length));
                     for (int i = 0; i < actual.getHits().getHits().length; i++) {
                         assertThat(actual.getHits().getAt(i).getId(), equalTo(expected.getHits().getAt(i).getId()));
@@ -231,7 +231,7 @@ public void testDuel() throws Exception {
                 prepareSearch("test").addSort("id", SortOrder.ASC)
                     .setQuery(QueryBuilders.boolQuery().should(QueryBuilders.termQuery("field2", "value"))),
                 expected -> {
-                    assertThat(actual.getHits().getTotalHits().value, equalTo(expected.getHits().getTotalHits().value));
+                    assertThat(actual.getHits().getTotalHits().value(), equalTo(expected.getHits().getTotalHits().value()));
                     assertThat(actual.getHits().getHits().length, equalTo(expected.getHits().getHits().length));
                     for (int i = 0; i < actual.getHits().getHits().length; i++) {
                         assertThat(actual.getHits().getAt(i).getId(), equalTo(expected.getHits().getAt(i).getId()));
@@ -254,7 +254,7 @@ public void testDuel() throws Exception {
                 prepareSearch("test").addSort("id", SortOrder.ASC)
                     .setQuery(QueryBuilders.boolQuery().should(QueryBuilders.termQuery("field3", "value"))),
                 expected -> {
-                    assertThat(actual.getHits().getTotalHits().value, equalTo(expected.getHits().getTotalHits().value));
+                    assertThat(actual.getHits().getTotalHits().value(), equalTo(expected.getHits().getTotalHits().value()));
                     assertThat(actual.getHits().getHits().length, equalTo(expected.getHits().getHits().length));
                     for (int i = 0; i < actual.getHits().getHits().length; i++) {
                         assertThat(actual.getHits().getAt(i).getId(), equalTo(expected.getHits().getAt(i).getId()));
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
index bffa53b1f4da6..66c8c0a5b1b52 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
@@ -956,10 +956,10 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(1));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(1));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                 }
@@ -975,10 +975,10 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(1));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(1));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
                 }
@@ -993,11 +993,11 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
@@ -1013,9 +1013,9 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(0));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(0));
                 }
             );
@@ -1029,12 +1029,12 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(3));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field3"), is("value3"));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(3));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
@@ -1051,12 +1051,12 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(3));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field3"), is("value3"));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(3));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
@@ -1073,12 +1073,12 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(3));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field3"), is("value3"));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(3));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
@@ -1095,11 +1095,11 @@ public void testMSearchApi() throws Exception {
                     .add(prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery())),
                 response -> {
                     assertFalse(response.getResponses()[0].isFailure());
-                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[0].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
-                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value, is(1L));
+                    assertThat(response.getResponses()[1].getResponse().getHits().getTotalHits().value(), is(1L));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().size(), is(2));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
                     assertThat(response.getResponses()[1].getResponse().getHits().getAt(0).getSourceAsMap().get("field2"), is("value2"));
@@ -1132,7 +1132,7 @@ public void testScroll() throws Exception {
                 .get();
 
             do {
-                assertThat(response.getHits().getTotalHits().value, is((long) numDocs));
+                assertThat(response.getHits().getTotalHits().value(), is((long) numDocs));
                 assertThat(response.getHits().getHits().length, is(1));
                 assertThat(response.getHits().getAt(0).getSourceAsMap().size(), is(1));
                 assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
@@ -1191,7 +1191,7 @@ public void testPointInTimeId() throws Exception {
                         .setQuery(constantScoreQuery(termQuery("field1", "value1")))
                         .setFetchSource(true),
                     response -> {
-                        assertThat(response.getHits().getTotalHits().value, is((long) numDocs));
+                        assertThat(response.getHits().getTotalHits().value(), is((long) numDocs));
                         assertThat(response.getHits().getHits().length, is(1));
                         assertThat(response.getHits().getAt(0).getSourceAsMap().size(), is(1));
                         assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
@@ -1281,7 +1281,7 @@ public void testScrollWithQueryCache() {
                             .setSize(1)
                             .setFetchSource(true)
                             .get();
-                        assertThat(user2SearchResponse.getHits().getTotalHits().value, is((long) 0));
+                        assertThat(user2SearchResponse.getHits().getTotalHits().value(), is((long) 0));
                         assertThat(user2SearchResponse.getHits().getHits().length, is(0));
                     } else {
                         user2SearchResponse.decRef();
@@ -1289,7 +1289,7 @@ public void testScrollWithQueryCache() {
                         user2SearchResponse = client().filterWithHeader(
                             Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))
                         ).prepareSearchScroll(user2SearchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(10L)).get();
-                        assertThat(user2SearchResponse.getHits().getTotalHits().value, is((long) 0));
+                        assertThat(user2SearchResponse.getHits().getTotalHits().value(), is((long) 0));
                         assertThat(user2SearchResponse.getHits().getHits().length, is(0));
                         if (randomBoolean()) {
                             // maybe reuse the scroll even if empty
@@ -1309,7 +1309,7 @@ public void testScrollWithQueryCache() {
                             .setSize(1)
                             .setFetchSource(true)
                             .get();
-                        assertThat(user1SearchResponse.getHits().getTotalHits().value, is((long) numDocs));
+                        assertThat(user1SearchResponse.getHits().getTotalHits().value(), is((long) numDocs));
                         assertThat(user1SearchResponse.getHits().getHits().length, is(1));
                         assertThat(user1SearchResponse.getHits().getAt(0).getSourceAsMap().size(), is(1));
                         assertThat(user1SearchResponse.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
@@ -1319,7 +1319,7 @@ public void testScrollWithQueryCache() {
                         user1SearchResponse = client().filterWithHeader(
                             Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))
                         ).prepareSearchScroll(user1SearchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(10L)).get();
-                        assertThat(user1SearchResponse.getHits().getTotalHits().value, is((long) numDocs));
+                        assertThat(user1SearchResponse.getHits().getTotalHits().value(), is((long) numDocs));
                         if (scrolledDocsUser1 < numDocs) {
                             assertThat(user1SearchResponse.getHits().getHits().length, is(1));
                             assertThat(user1SearchResponse.getHits().getAt(0).getSourceAsMap().size(), is(1));
@@ -2042,7 +2042,7 @@ private void verifyParentChild() {
                 .setQuery(hasChildQuery("child", termQuery("field1", "yellow"), ScoreMode.None)),
             searchResponse -> {
                 assertHitCount(searchResponse, 1L);
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("p1"));
             }
         );
@@ -2061,7 +2061,7 @@ private void verifyParentChild() {
                 .setQuery(hasChildQuery("child", termQuery("alias", "yellow"), ScoreMode.None)),
             searchResponse -> {
                 assertHitCount(searchResponse, 1L);
-                assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("p1"));
             }
         );
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
index d4375d15e6a6d..7d99d5817bdc0 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
@@ -103,20 +103,20 @@ public void testSearchAndMSearch() throws Exception {
         indexRandom(true, prepareIndex(index).setSource(field, "bar"));
 
         assertResponse(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), response -> {
-            final long hits = response.getHits().getTotalHits().value;
+            final long hits = response.getHits().getTotalHits().value();
             assertThat(hits, greaterThan(0L));
             assertResponse(
                 client().filterWithHeader(
                     singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD))
                 ).prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()),
-                response2 -> assertEquals(response2.getHits().getTotalHits().value, hits)
+                response2 -> assertEquals(response2.getHits().getTotalHits().value(), hits)
             );
             final long multiHits;
             MultiSearchResponse multiSearchResponse = client().prepareMultiSearch()
                 .add(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()))
                 .get();
             try {
-                multiHits = multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits().value;
+                multiHits = multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits().value();
                 assertThat(hits, greaterThan(0L));
             } finally {
                 multiSearchResponse.decRef();
@@ -125,7 +125,7 @@ public void testSearchAndMSearch() throws Exception {
                 singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD))
             ).prepareMultiSearch().add(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery())).get();
             try {
-                assertEquals(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits().value, multiHits);
+                assertEquals(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits().value(), multiHits);
             } finally {
                 multiSearchResponse.decRef();
             }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
index af54f71779f08..6f8ea0f103a56 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
@@ -312,7 +312,7 @@ public void testMultiNamesWorkCorrectly() {
 
         assertResponse(
             userAClient.prepareSearch("alias1").setSize(0),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L))
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(0L))
         );
 
         final ElasticsearchSecurityException e1 = expectThrows(
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
index 778d88d832887..fdd854e7a9673 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
@@ -45,6 +46,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -63,7 +65,6 @@
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
@@ -154,7 +155,17 @@ public void cleanUp() {
     }
 
     public static void writeJSONFile(String node, String json, Logger logger, AtomicLong versionCounter) throws Exception {
-        long version = versionCounter.incrementAndGet();
+        writeJSONFile(node, json, logger, versionCounter, true);
+    }
+
+    public static void writeJSONFileWithoutVersionIncrement(String node, String json, Logger logger, AtomicLong versionCounter)
+        throws Exception {
+        writeJSONFile(node, json, logger, versionCounter, false);
+    }
+
+    private static void writeJSONFile(String node, String json, Logger logger, AtomicLong versionCounter, boolean incrementVersion)
+        throws Exception {
+        long version = incrementVersion ? versionCounter.incrementAndGet() : versionCounter.get();
 
         FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
         assertTrue(fileSettingsService.watching());
@@ -261,21 +272,28 @@ private void assertRoleMappingsSaveOK(CountDownLatch savedClusterState, AtomicLo
             assertThat(resolveRolesFuture.get(), containsInAnyOrder("kibana_user", "fleet_user"));
         }
 
-        // the role mappings are not retrievable by the role mapping action (which only accesses "native" i.e. index-based role mappings)
-        var request = new GetRoleMappingsRequest();
-        request.setNames("everyone_kibana", "everyone_fleet");
-        var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
-        assertFalse(response.hasMappings());
-        assertThat(response.mappings(), emptyArray());
-
-        // role mappings (with the same names) can also be stored in the "native" store
-        var putRoleMappingResponse = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_kibana")).actionGet();
-        assertTrue(putRoleMappingResponse.isCreated());
-        putRoleMappingResponse = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_fleet")).actionGet();
-        assertTrue(putRoleMappingResponse.isCreated());
+        // the role mappings are retrievable by the role mapping action for BWC
+        assertGetResponseHasMappings(true, "everyone_kibana", "everyone_fleet");
+
+        // role mappings (with the same names) can be stored in the "native" store
+        {
+            PutRoleMappingResponse response = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_kibana"))
+                .actionGet();
+            assertTrue(response.isCreated());
+            response = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_fleet")).actionGet();
+            assertTrue(response.isCreated());
+        }
+        {
+            // deleting role mappings that exist in the native store and in cluster-state should result in success
+            var response = client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).actionGet();
+            assertTrue(response.isFound());
+            response = client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_fleet")).actionGet();
+            assertTrue(response.isFound());
+        }
+
     }
 
-    public void testRoleMappingsApplied() throws Exception {
+    public void testClusterStateRoleMappingsAddedThenDeleted() throws Exception {
         ensureGreen();
 
         var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana");
@@ -284,6 +302,12 @@ public void testRoleMappingsApplied() throws Exception {
         assertRoleMappingsSaveOK(savedClusterState.v1(), savedClusterState.v2());
         logger.info("---> cleanup cluster settings...");
 
+        {
+            // Deleting non-existent native role mappings returns not found even if they exist in config file
+            var response = client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).get();
+            assertFalse(response.isFound());
+        }
+
         savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName());
 
         writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter);
@@ -298,48 +322,96 @@ public void testRoleMappingsApplied() throws Exception {
             clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey())
         );
 
-        // native role mappings are not affected by the removal of the cluster-state based ones
+        // cluster-state role mapping was removed and is not returned in the API anymore
         {
             var request = new GetRoleMappingsRequest();
             request.setNames("everyone_kibana", "everyone_fleet");
             var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
-            assertTrue(response.hasMappings());
-            assertThat(
-                Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(),
-                containsInAnyOrder("everyone_kibana", "everyone_fleet")
-            );
+            assertFalse(response.hasMappings());
         }
 
-        // and roles are resolved based on the native role mappings
+        // no role mappings means no roles are resolved
         for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) {
             PlainActionFuture<Set<String>> resolveRolesFuture = new PlainActionFuture<>();
             userRoleMapper.resolveRoles(
                 new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)),
                 resolveRolesFuture
             );
-            assertThat(resolveRolesFuture.get(), contains("kibana_user_native"));
+            assertThat(resolveRolesFuture.get(), empty());
         }
+    }
 
-        {
-            var request = new DeleteRoleMappingRequest();
-            request.setName("everyone_kibana");
-            var response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get();
-            assertTrue(response.isFound());
-            request = new DeleteRoleMappingRequest();
-            request.setName("everyone_fleet");
-            response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get();
-            assertTrue(response.isFound());
+    public void testGetRoleMappings() throws Exception {
+        ensureGreen();
+
+        final List<String> nativeMappings = List.of("everyone_kibana", "_everyone_kibana", "zzz_mapping", "123_mapping");
+        for (var mapping : nativeMappings) {
+            client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest(mapping)).actionGet();
         }
 
-        // no roles are resolved now, because both native and cluster-state based stores have been cleared
-        for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) {
-            PlainActionFuture<Set<String>> resolveRolesFuture = new PlainActionFuture<>();
-            userRoleMapper.resolveRoles(
-                new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)),
-                resolveRolesFuture
-            );
-            assertThat(resolveRolesFuture.get(), empty());
+        var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana");
+        writeJSONFile(internalCluster().getMasterName(), testJSON, logger, versionCounter);
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        var request = new GetRoleMappingsRequest();
+        var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
+        assertTrue(response.hasMappings());
+        assertThat(
+            Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(),
+            containsInAnyOrder(
+                "everyone_kibana",
+                ExpressionRoleMapping.addReadOnlySuffix("everyone_kibana"),
+                "_everyone_kibana",
+                ExpressionRoleMapping.addReadOnlySuffix("everyone_fleet"),
+                "zzz_mapping",
+                "123_mapping"
+            )
+        );
+
+        List<Boolean> readOnlyFlags = new ArrayList<>();
+        for (ExpressionRoleMapping mapping : response.mappings()) {
+            boolean isReadOnly = ExpressionRoleMapping.hasReadOnlySuffix(mapping.getName())
+                && mapping.getMetadata().get("_read_only") != null;
+            readOnlyFlags.add(isReadOnly);
         }
+        // assert that cluster-state role mappings come last
+        assertThat(readOnlyFlags, contains(false, false, false, false, true, true));
+
+        // it's possible to delete overlapping native role mapping
+        assertTrue(client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).actionGet().isFound());
+
+        // Fetch a specific file based role
+        request = new GetRoleMappingsRequest();
+        request.setNames(ExpressionRoleMapping.addReadOnlySuffix("everyone_kibana"));
+        response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
+        assertTrue(response.hasMappings());
+        assertThat(
+            Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(),
+            containsInAnyOrder(ExpressionRoleMapping.addReadOnlySuffix("everyone_kibana"))
+        );
+
+        savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName());
+        writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter);
+        awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        final ClusterStateResponse clusterStateResponse = clusterAdmin().state(
+            new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(savedClusterState.v2().get())
+        ).get();
+
+        assertNull(
+            clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey())
+        );
+
+        // Make sure remaining native mappings can still be fetched
+        request = new GetRoleMappingsRequest();
+        response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
+        assertTrue(response.hasMappings());
+        assertThat(
+            Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(),
+            containsInAnyOrder("_everyone_kibana", "zzz_mapping", "123_mapping")
+        );
     }
 
     public static Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(
@@ -424,11 +496,8 @@ public void testRoleMappingApplyWithSecurityIndexClosed() throws Exception {
             boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
             assertTrue(awaitSuccessful);
 
-            // no native role mappings exist
-            var request = new GetRoleMappingsRequest();
-            request.setNames("everyone_kibana", "everyone_fleet");
-            var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
-            assertFalse(response.hasMappings());
+            // even if index is closed, cluster-state role mappings are still returned
+            assertGetResponseHasMappings(true, "everyone_kibana", "everyone_fleet");
 
             // cluster state settings are also applied
             var clusterStateResponse = clusterAdmin().state(
@@ -467,6 +536,12 @@ public void testRoleMappingApplyWithSecurityIndexClosed() throws Exception {
         }
     }
 
+    private DeleteRoleMappingRequest deleteRequest(String name) {
+        var request = new DeleteRoleMappingRequest();
+        request.setName(name);
+        return request;
+    }
+
     private PutRoleMappingRequest sampleRestRequest(String name) throws Exception {
         var json = """
             {
@@ -485,4 +560,19 @@ private PutRoleMappingRequest sampleRestRequest(String name) throws Exception {
             return new PutRoleMappingRequestBuilder(null).source(name, parser).request();
         }
     }
+
+    private static void assertGetResponseHasMappings(boolean readOnly, String... mappings) throws InterruptedException, ExecutionException {
+        var request = new GetRoleMappingsRequest();
+        request.setNames(mappings);
+        var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
+        assertTrue(response.hasMappings());
+        assertThat(
+            Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(),
+            containsInAnyOrder(
+                Arrays.stream(mappings)
+                    .map(mapping -> readOnly ? ExpressionRoleMapping.addReadOnlySuffix(mapping) : mapping)
+                    .toArray(String[]::new)
+            )
+        );
+    }
 }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
index c0f82adc88784..97a5f080cee4e 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
@@ -16,25 +16,34 @@
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
 import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
 import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
+import org.junit.Before;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListener;
 import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListenerForCleanup;
 import static org.elasticsearch.integration.RoleMappingFileSettingsIT.writeJSONFile;
+import static org.elasticsearch.integration.RoleMappingFileSettingsIT.writeJSONFileWithoutVersionIncrement;
+import static org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata.METADATA_NAME_FIELD;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.emptyIterable;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase {
 
-    private static AtomicLong versionCounter = new AtomicLong(1);
+    private final AtomicLong versionCounter = new AtomicLong(1);
 
-    private static String testJSONOnlyRoleMappings = """
+    @Before
+    public void resetVersion() {
+        versionCounter.set(1);
+    }
+
+    private static final String testJSONOnlyRoleMappings = """
         {
              "metadata": {
                  "version": "%s",
@@ -64,7 +73,28 @@ public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase {
              }
         }""";
 
-    private static String emptyJSON = """
+    private static final String testJSONOnlyUpdatedRoleMappings = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "everyone_kibana_together": {
+                          "enabled": true,
+                          "roles": [ "kibana_user", "kibana_admin" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                             "_foo": "something"
+                          }
+                       }
+                 }
+             }
+        }""";
+
+    private static final String emptyJSON = """
         {
              "metadata": {
                  "version": "%s",
@@ -88,84 +118,196 @@ public void testReservedStatePersistsOnRestart() throws Exception {
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
-        var clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState();
-        assertRoleMappingReservedMetadata(clusterState, "everyone_kibana_alone", "everyone_fleet_alone");
-        List<ExpressionRoleMapping> roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
-        assertThat(
-            roleMappings,
-            containsInAnyOrder(
-                new ExpressionRoleMapping(
-                    "everyone_kibana_alone",
-                    new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
-                    List.of("kibana_user"),
-                    List.of(),
-                    Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"),
-                    true
+        assertRoleMappingsInClusterState(
+            new ExpressionRoleMapping(
+                "everyone_kibana_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("kibana_user"),
+                List.of(),
+                Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something", METADATA_NAME_FIELD, "everyone_kibana_alone"),
+                true
+            ),
+            new ExpressionRoleMapping(
+                "everyone_fleet_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("fleet_user"),
+                List.of(),
+                Map.of(
+                    "uuid",
+                    "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7",
+                    "_foo",
+                    "something_else",
+                    METADATA_NAME_FIELD,
+                    "everyone_fleet_alone"
                 ),
-                new ExpressionRoleMapping(
-                    "everyone_fleet_alone",
-                    new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
-                    List.of("fleet_user"),
-                    List.of(),
-                    Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"),
-                    false
-                )
+                false
             )
         );
 
         logger.info("--> restart master");
         internalCluster().restartNode(masterNode);
         ensureGreen();
+        awaitFileSettingsWatcher();
 
-        // assert role mappings are recovered from "disk"
-        clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState();
-        assertRoleMappingReservedMetadata(clusterState, "everyone_kibana_alone", "everyone_fleet_alone");
-        roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
-        assertThat(
-            roleMappings,
-            containsInAnyOrder(
-                new ExpressionRoleMapping(
-                    "name_not_available_after_deserialization",
-                    new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
-                    List.of("kibana_user"),
-                    List.of(),
-                    Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"),
-                    true
+        assertRoleMappingsInClusterState(
+            new ExpressionRoleMapping(
+                "everyone_kibana_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("kibana_user"),
+                List.of(),
+                Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something", METADATA_NAME_FIELD, "everyone_kibana_alone"),
+                true
+            ),
+            new ExpressionRoleMapping(
+                "everyone_fleet_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("fleet_user"),
+                List.of(),
+                Map.of(
+                    "uuid",
+                    "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7",
+                    "_foo",
+                    "something_else",
+                    METADATA_NAME_FIELD,
+                    "everyone_fleet_alone"
+                ),
+                false
+            )
+        );
+
+        // now remove the role mappings via the same settings file
+        cleanupClusterState(masterNode);
+
+        // no role mappings
+        assertRoleMappingsInClusterState();
+
+        // and restart the master to confirm the role mappings are all gone
+        logger.info("--> restart master again");
+        internalCluster().restartNode(masterNode);
+        ensureGreen();
+
+        // no role mappings
+        assertRoleMappingsInClusterState();
+    }
+
+    public void testFileSettingsReprocessedOnRestartWithoutVersionChange() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+
+        final String masterNode = internalCluster().getMasterName();
+
+        var savedClusterState = setupClusterStateListener(masterNode, "everyone_kibana_alone");
+        awaitFileSettingsWatcher();
+        logger.info("--> write some role mappings, no other file settings");
+        writeJSONFile(masterNode, testJSONOnlyRoleMappings, logger, versionCounter);
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        assertRoleMappingsInClusterState(
+            new ExpressionRoleMapping(
+                "everyone_kibana_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("kibana_user"),
+                List.of(),
+                Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something", METADATA_NAME_FIELD, "everyone_kibana_alone"),
+                true
+            ),
+            new ExpressionRoleMapping(
+                "everyone_fleet_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("fleet_user"),
+                List.of(),
+                Map.of(
+                    "uuid",
+                    "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7",
+                    "_foo",
+                    "something_else",
+                    METADATA_NAME_FIELD,
+                    "everyone_fleet_alone"
+                ),
+                false
+            )
+        );
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        final FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+        fileSettingsService.addFileChangedListener(latch::countDown);
+        // Don't increment version but write new file contents to test re-processing on restart
+        writeJSONFileWithoutVersionIncrement(masterNode, testJSONOnlyUpdatedRoleMappings, logger, versionCounter);
+        // Make sure we saw a file settings update so that we know it got processed, but it did not affect cluster state
+        assertTrue(latch.await(20, TimeUnit.SECONDS));
+
+        // Nothing changed yet because version is the same and there was no restart
+        assertRoleMappingsInClusterState(
+            new ExpressionRoleMapping(
+                "everyone_kibana_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("kibana_user"),
+                List.of(),
+                Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something", METADATA_NAME_FIELD, "everyone_kibana_alone"),
+                true
+            ),
+            new ExpressionRoleMapping(
+                "everyone_fleet_alone",
+                new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                List.of("fleet_user"),
+                List.of(),
+                Map.of(
+                    "uuid",
+                    "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7",
+                    "_foo",
+                    "something_else",
+                    METADATA_NAME_FIELD,
+                    "everyone_fleet_alone"
                 ),
+                false
+            )
+        );
+
+        logger.info("--> restart master");
+        internalCluster().restartNode(masterNode);
+        ensureGreen();
+        awaitFileSettingsWatcher();
+
+        // Assert busy to give mappings time to update
+        assertBusy(
+            () -> assertRoleMappingsInClusterState(
                 new ExpressionRoleMapping(
-                    "name_not_available_after_deserialization",
+                    "everyone_kibana_together",
                     new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
-                    List.of("fleet_user"),
+                    List.of("kibana_user", "kibana_admin"),
                     List.of(),
-                    Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"),
-                    false
+                    Map.of(
+                        "uuid",
+                        "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                        "_foo",
+                        "something",
+                        METADATA_NAME_FIELD,
+                        "everyone_kibana_together"
+                    ),
+                    true
                 )
             )
         );
 
+        cleanupClusterState(masterNode);
+    }
+
+    private void assertRoleMappingsInClusterState(ExpressionRoleMapping... expectedRoleMappings) {
+        var clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState();
+        String[] expectedRoleMappingNames = Arrays.stream(expectedRoleMappings).map(ExpressionRoleMapping::getName).toArray(String[]::new);
+        assertRoleMappingReservedMetadata(clusterState, expectedRoleMappingNames);
+        var actualRoleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
+        assertThat(actualRoleMappings, containsInAnyOrder(expectedRoleMappings));
+    }
+
+    private void cleanupClusterState(String masterNode) throws Exception {
         // now remove the role mappings via the same settings file
-        savedClusterState = setupClusterStateListenerForCleanup(masterNode);
+        var savedClusterState = setupClusterStateListenerForCleanup(masterNode);
         awaitFileSettingsWatcher();
         logger.info("--> remove the role mappings with an empty settings file");
         writeJSONFile(masterNode, emptyJSON, logger, versionCounter);
-        awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
-
-        clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState();
-        assertRoleMappingReservedMetadata(clusterState);
-        roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
-        assertThat(roleMappings, emptyIterable());
-
-        // and restart the master to confirm the role mappings are all gone
-        logger.info("--> restart master again");
-        internalCluster().restartNode(masterNode);
-        ensureGreen();
-
-        // assert empty role mappings are recovered from "disk"
-        clusterState = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet().getState();
-        assertRoleMappingReservedMetadata(clusterState);
-        roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
-        assertThat(roleMappings, emptyIterable());
     }
 
     private void assertRoleMappingReservedMetadata(ClusterState clusterState, String... names) {
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
index 78146e58e91e2..e178f4bf3eb6c 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
@@ -343,7 +343,7 @@ private void testAddUserAndRoleThenAuth(String username, String roleName) {
         String token = basicAuthHeaderValue(username, new SecureString("s3krit-password"));
         assertResponse(
             client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx"),
-            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value)
+            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value())
         );
 
         assertClusterHealthOnlyAuthorizesWhenAnonymousRoleActive(token);
@@ -366,7 +366,7 @@ public void testUpdatingUserAndAuthentication() throws Exception {
         String token = basicAuthHeaderValue("joe", new SecureString("s3krit-password"));
         assertResponse(
             client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx"),
-            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value)
+            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value())
         );
 
         preparePutUser("joe", "s3krit-password2", hasher, SecuritySettingsSource.TEST_ROLE).get();
@@ -382,7 +382,7 @@ public void testUpdatingUserAndAuthentication() throws Exception {
         token = basicAuthHeaderValue("joe", new SecureString("s3krit-password2"));
         assertResponse(
             client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx"),
-            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value)
+            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value())
         );
     }
 
@@ -403,7 +403,7 @@ public void testCreateDeleteAuthenticate() {
         String token = basicAuthHeaderValue("joe", new SecureString("s3krit-password"));
         assertResponse(
             client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx"),
-            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value)
+            searchResp -> assertEquals(1L, searchResp.getHits().getTotalHits().value())
         );
 
         DeleteUserResponse response = new DeleteUserRequestBuilder(client()).username("joe").get();
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
index f34983f7f125c..0acc281dd8440 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
@@ -213,7 +213,7 @@ public void testMultiSearchUnauthorizedIndex() {
                     assertEquals(2, multiSearchResponse.getResponses().length);
                     assertFalse(multiSearchResponse.getResponses()[0].isFailure());
                     SearchResponse searchResponse = multiSearchResponse.getResponses()[0].getResponse();
-                    assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+                    assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
                     assertReturnedIndices(searchResponse, "test1", "test2", "test3");
                     assertTrue(multiSearchResponse.getResponses()[1].isFailure());
                     Exception exception = multiSearchResponse.getResponses()[1].getFailure();
@@ -231,7 +231,7 @@ public void testMultiSearchUnauthorizedIndex() {
                     assertEquals(2, multiSearchResponse.getResponses().length);
                     assertFalse(multiSearchResponse.getResponses()[0].isFailure());
                     SearchResponse searchResponse = multiSearchResponse.getResponses()[0].getResponse();
-                    assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+                    assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
                     assertReturnedIndices(searchResponse, "test1", "test2", "test3");
                     assertFalse(multiSearchResponse.getResponses()[1].isFailure());
                     assertNoSearchHits(multiSearchResponse.getResponses()[1].getResponse());
@@ -249,7 +249,7 @@ public void testMultiSearchMissingUnauthorizedIndex() {
                     assertEquals(2, multiSearchResponse.getResponses().length);
                     assertFalse(multiSearchResponse.getResponses()[0].isFailure());
                     SearchResponse searchResponse = multiSearchResponse.getResponses()[0].getResponse();
-                    assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+                    assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
                     assertReturnedIndices(searchResponse, "test1", "test2", "test3");
                     assertTrue(multiSearchResponse.getResponses()[1].isFailure());
                     Exception exception = multiSearchResponse.getResponses()[1].getFailure();
@@ -267,7 +267,7 @@ public void testMultiSearchMissingUnauthorizedIndex() {
                     assertEquals(2, multiSearchResponse.getResponses().length);
                     assertFalse(multiSearchResponse.getResponses()[0].isFailure());
                     SearchResponse searchResponse = multiSearchResponse.getResponses()[0].getResponse();
-                    assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+                    assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
                     assertReturnedIndices(searchResponse, "test1", "test2", "test3");
                     assertFalse(multiSearchResponse.getResponses()[1].isFailure());
                     assertNoSearchHits(multiSearchResponse.getResponses()[1].getResponse());
@@ -317,7 +317,7 @@ public void testMultiSearchWildcard() {
                     assertEquals(2, multiSearchResponse.getResponses().length);
                     assertFalse(multiSearchResponse.getResponses()[0].isFailure());
                     SearchResponse searchResponse = multiSearchResponse.getResponses()[0].getResponse();
-                    assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+                    assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
                     assertReturnedIndices(searchResponse, "test1", "test2", "test3");
                     assertNoSearchHits(multiSearchResponse.getResponses()[1].getResponse());
                 }
@@ -336,7 +336,7 @@ public void testMultiSearchWildcard() {
                     assertEquals(2, multiSearchResponse.getResponses().length);
                     assertFalse(multiSearchResponse.getResponses()[0].isFailure());
                     SearchResponse searchResponse = multiSearchResponse.getResponses()[0].getResponse();
-                    assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+                    assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
                     assertReturnedIndices(searchResponse, "test1", "test2", "test3");
                     assertTrue(multiSearchResponse.getResponses()[1].isFailure());
                     Exception exception = multiSearchResponse.getResponses()[1].getFailure();
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java
index eb7c5e5276c15..a4cadeb953e14 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java
@@ -48,13 +48,13 @@ public void testScrollIsPerUser() throws Exception {
         indexRandom(true, docs);
 
         assertResponse(prepareSearch("foo").setScroll(TimeValue.timeValueSeconds(5L)).setQuery(matchAllQuery()).setSize(1), response -> {
-            assertEquals(numDocs, response.getHits().getTotalHits().value);
+            assertEquals(numDocs, response.getHits().getTotalHits().value());
             assertEquals(1, response.getHits().getHits().length);
             if (randomBoolean()) {
                 assertResponse(
                     client().prepareSearchScroll(response.getScrollId()).setScroll(TimeValue.timeValueSeconds(5L)),
                     response2 -> {
-                        assertEquals(numDocs, response2.getHits().getTotalHits().value);
+                        assertEquals(numDocs, response2.getHits().getTotalHits().value());
                         assertEquals(1, response2.getHits().getHits().length);
                     }
                 );
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java
index 4b8fbfd41acdf..437fb76351176 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java
@@ -856,7 +856,7 @@ private SuggestProfilesResponse.ProfileHit[] doSuggest(Set<String> dataKeys, Str
         final SuggestProfilesRequest suggestProfilesRequest = new SuggestProfilesRequest(dataKeys, name, 10, hint);
         final SuggestProfilesResponse suggestProfilesResponse = client().execute(SuggestProfilesAction.INSTANCE, suggestProfilesRequest)
             .actionGet();
-        assertThat(suggestProfilesResponse.getTotalHits().relation, is(TotalHits.Relation.EQUAL_TO));
+        assertThat(suggestProfilesResponse.getTotalHits().relation(), is(TotalHits.Relation.EQUAL_TO));
         return suggestProfilesResponse.getProfileHits();
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 79a00fa1293bd..0b387a738a2c5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -810,13 +810,11 @@ Collection<Object> createComponents(
         // We need to construct the checks here while the secure settings are still available.
         // If we wait until #getBoostrapChecks the secure settings will have been cleared/closed.
         final List<BootstrapCheck> checks = new ArrayList<>();
-        checks.addAll(
-            Arrays.asList(
-                new TokenSSLBootstrapCheck(),
-                new PkiRealmBootstrapCheck(getSslService()),
-                new SecurityImplicitBehaviorBootstrapCheck(nodeMetadata, getLicenseService()),
-                new TransportTLSBootstrapCheck()
-            )
+        Collections.addAll(
+            checks,
+            new TokenSSLBootstrapCheck(),
+            new PkiRealmBootstrapCheck(getSslService()),
+            new TransportTLSBootstrapCheck()
         );
         checks.addAll(InternalRealms.getBootstrapChecks(settings, environment));
         this.bootstrapChecks.set(Collections.unmodifiableList(checks));
@@ -899,6 +897,7 @@ Collection<Object> createComponents(
         components.add(nativeUsersStore);
         components.add(new PluginComponentBinding<>(NativeRoleMappingStore.class, nativeRoleMappingStore));
         components.add(new PluginComponentBinding<>(UserRoleMapper.class, userRoleMapper));
+        components.add(clusterStateRoleMapper);
         components.add(reservedRealm);
         components.add(realms);
         this.realms.set(realms);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheck.java
deleted file mode 100644
index 2d535100d468d..0000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheck.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.security;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.BootstrapCheck;
-import org.elasticsearch.bootstrap.BootstrapContext;
-import org.elasticsearch.common.ReferenceDocs;
-import org.elasticsearch.env.NodeMetadata;
-import org.elasticsearch.license.ClusterStateLicenseService;
-import org.elasticsearch.license.License;
-import org.elasticsearch.license.LicenseService;
-import org.elasticsearch.xpack.core.XPackSettings;
-
-public class SecurityImplicitBehaviorBootstrapCheck implements BootstrapCheck {
-
-    private final NodeMetadata nodeMetadata;
-    private final LicenseService licenseService;
-
-    public SecurityImplicitBehaviorBootstrapCheck(NodeMetadata nodeMetadata, LicenseService licenseService) {
-        this.nodeMetadata = nodeMetadata;
-        this.licenseService = licenseService;
-    }
-
-    @Override
-    public BootstrapCheckResult check(BootstrapContext context) {
-        if (nodeMetadata == null) {
-            return BootstrapCheckResult.success();
-        }
-        if (licenseService instanceof ClusterStateLicenseService clusterStateLicenseService) {
-            final License license = clusterStateLicenseService.getLicense(context.metadata());
-            // TODO[wrb]: Add an "isCurrentMajor" method to BuildVersion?
-            final Version lastKnownVersion = nodeMetadata.previousNodeVersion().toVersion();
-            // pre v7.2.0 nodes have Version.EMPTY and its id is 0, so Version#before handles this successfully
-            if (lastKnownVersion.before(Version.V_8_0_0)
-                && XPackSettings.SECURITY_ENABLED.exists(context.settings()) == false
-                && (license.operationMode() == License.OperationMode.BASIC || license.operationMode() == License.OperationMode.TRIAL)) {
-                return BootstrapCheckResult.failure(
-                    "The default value for ["
-                        + XPackSettings.SECURITY_ENABLED.getKey()
-                        + "] has changed in the current version. "
-                        + " Security features were implicitly disabled for this node but they would now be enabled, possibly"
-                        + " preventing access to the node. "
-                        + "See "
-                        + this.referenceDocs()
-                        + " to configure security, or explicitly disable security by "
-                        + "setting [xpack.security.enabled] to \"false\" in elasticsearch.yml before restarting the node."
-                );
-            }
-        }
-        return BootstrapCheckResult.success();
-    }
-
-    public boolean alwaysEnforce() {
-        return true;
-    }
-
-    @Override
-    public ReferenceDocs referenceDocs() {
-        return ReferenceDocs.BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP;
-    }
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java
index 73d1a1abcdb50..837b475dea68f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java
@@ -43,7 +43,7 @@ public String name() {
     @Override
     public TransformState transform(Object source, TransformState prevState) throws Exception {
         @SuppressWarnings("unchecked")
-        Set<ExpressionRoleMapping> roleMappings = validate((List<PutRoleMappingRequest>) source);
+        Set<ExpressionRoleMapping> roleMappings = validateAndTranslate((List<PutRoleMappingRequest>) source);
         RoleMappingMetadata newRoleMappingMetadata = new RoleMappingMetadata(roleMappings);
         if (newRoleMappingMetadata.equals(RoleMappingMetadata.getFromClusterState(prevState.state()))) {
             return prevState;
@@ -71,7 +71,7 @@ public List<PutRoleMappingRequest> fromXContent(XContentParser parser) throws IO
         return result;
     }
 
-    private Set<ExpressionRoleMapping> validate(List<PutRoleMappingRequest> roleMappings) {
+    private Set<ExpressionRoleMapping> validateAndTranslate(List<PutRoleMappingRequest> roleMappings) {
         var exceptions = new ArrayList<Exception>();
         for (var roleMapping : roleMappings) {
             // File based defined role mappings are allowed to use MetadataUtils.RESERVED_PREFIX
@@ -85,6 +85,8 @@ private Set<ExpressionRoleMapping> validate(List<PutRoleMappingRequest> roleMapp
             exceptions.forEach(illegalArgumentException::addSuppressed);
             throw illegalArgumentException;
         }
-        return roleMappings.stream().map(PutRoleMappingRequest::getMapping).collect(Collectors.toUnmodifiableSet());
+        return roleMappings.stream()
+            .map(r -> RoleMappingMetadata.copyWithNameInMetadata(r.getMapping()))
+            .collect(Collectors.toUnmodifiableSet());
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java
index 74129facae70a..b1fdf2e90dd46 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
@@ -16,17 +17,19 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse;
+import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 
 public class TransportDeleteRoleMappingAction extends HandledTransportAction<DeleteRoleMappingRequest, DeleteRoleMappingResponse> {
-
     private final NativeRoleMappingStore roleMappingStore;
+    private final ClusterStateRoleMapper clusterStateRoleMapper;
 
     @Inject
     public TransportDeleteRoleMappingAction(
         ActionFilters actionFilters,
         TransportService transportService,
-        NativeRoleMappingStore roleMappingStore
+        NativeRoleMappingStore roleMappingStore,
+        ClusterStateRoleMapper clusterStateRoleMapper
     ) {
         super(
             DeleteRoleMappingAction.NAME,
@@ -36,10 +39,24 @@ public TransportDeleteRoleMappingAction(
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.roleMappingStore = roleMappingStore;
+        this.clusterStateRoleMapper = clusterStateRoleMapper;
     }
 
     @Override
     protected void doExecute(Task task, DeleteRoleMappingRequest request, ActionListener<DeleteRoleMappingResponse> listener) {
-        roleMappingStore.deleteRoleMapping(request, listener.safeMap(DeleteRoleMappingResponse::new));
+        roleMappingStore.deleteRoleMapping(request, listener.safeMap(found -> {
+            if (found && clusterStateRoleMapper.hasMapping(request.getName())) {
+                // Allow to delete a mapping with the same name in the native role mapping store as the file_settings namespace, but
+                // add a warning header to signal to the caller that this could be a problem.
+                HeaderWarning.addWarning(
+                    "A read-only role mapping with the same name ["
+                        + request.getName()
+                        + "] has previously been defined in a configuration file. "
+                        + "The native role mapping was deleted, but the read-only mapping will remain active "
+                        + "and will be used to determine role assignments."
+                );
+            }
+            return new DeleteRoleMappingResponse(found);
+        }));
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java
index ac0d3177cca09..5f16b095db0ef 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java
@@ -6,6 +6,8 @@
  */
 package org.elasticsearch.xpack.security.action.rolemapping;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -17,21 +19,31 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
+import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class TransportGetRoleMappingsAction extends HandledTransportAction<GetRoleMappingsRequest, GetRoleMappingsResponse> {
+    private static final Logger logger = LogManager.getLogger(TransportGetRoleMappingsAction.class);
 
     private final NativeRoleMappingStore roleMappingStore;
+    private final ClusterStateRoleMapper clusterStateRoleMapper;
 
     @Inject
     public TransportGetRoleMappingsAction(
         ActionFilters actionFilters,
         TransportService transportService,
-        NativeRoleMappingStore nativeRoleMappingStore
+        NativeRoleMappingStore nativeRoleMappingStore,
+        ClusterStateRoleMapper clusterStateRoleMapper
     ) {
         super(
             GetRoleMappingsAction.NAME,
@@ -41,19 +53,84 @@ public TransportGetRoleMappingsAction(
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.roleMappingStore = nativeRoleMappingStore;
+        this.clusterStateRoleMapper = clusterStateRoleMapper;
     }
 
     @Override
     protected void doExecute(Task task, final GetRoleMappingsRequest request, final ActionListener<GetRoleMappingsResponse> listener) {
         final Set<String> names;
         if (request.getNames() == null || request.getNames().length == 0) {
-            names = null;
+            names = Set.of();
         } else {
             names = new HashSet<>(Arrays.asList(request.getNames()));
         }
-        this.roleMappingStore.getRoleMappings(names, ActionListener.wrap(mappings -> {
-            ExpressionRoleMapping[] array = mappings.toArray(new ExpressionRoleMapping[mappings.size()]);
-            listener.onResponse(new GetRoleMappingsResponse(array));
+        roleMappingStore.getRoleMappings(names, ActionListener.wrap(nativeRoleMappings -> {
+            final Collection<ExpressionRoleMapping> clusterStateRoleMappings = clusterStateRoleMapper.getMappings(
+                // if the API was queried with a reserved suffix for any of the names, we need to remove it because role mappings are
+                // stored without it in cluster-state
+                removeReadOnlySuffixIfPresent(names)
+            );
+            listener.onResponse(buildResponse(clusterStateRoleMappings, nativeRoleMappings));
         }, listener::onFailure));
     }
+
+    private GetRoleMappingsResponse buildResponse(
+        Collection<ExpressionRoleMapping> clusterStateMappings,
+        Collection<ExpressionRoleMapping> nativeMappings
+    ) {
+        Stream<ExpressionRoleMapping> translatedClusterStateMappings = clusterStateMappings.stream().filter(roleMapping -> {
+            if (RoleMappingMetadata.hasFallbackName(roleMapping)) {
+                logger.warn(
+                    "Role mapping retrieved from cluster-state with an ambiguous name. It will be omitted from the API response."
+                        + "This is likely a transient issue during node start-up."
+                );
+                return false;
+            }
+            return true;
+        }).map(this::translateClusterStateMapping);
+        return new GetRoleMappingsResponse(
+            Stream.concat(nativeMappings.stream(), translatedClusterStateMappings).toArray(ExpressionRoleMapping[]::new)
+        );
+    }
+
+    private Set<String> removeReadOnlySuffixIfPresent(Set<String> names) {
+        return names.stream().map(ExpressionRoleMapping::removeReadOnlySuffixIfPresent).collect(Collectors.toSet());
+    }
+
+    /**
+     * Translator method for ensuring unique API names and marking cluster-state role mappings as read-only.
+     * Role mappings retrieved from cluster-state are surfaced through both the transport and REST layers,
+     * along with native role mappings. Unlike native role mappings, cluster-state role mappings are
+     * read-only and cannot be modified via APIs. It is possible for cluster-state and native role mappings
+     * to have overlapping names.
+     *
+     * <p>
+     * This does the following:
+     * </p>
+     *
+     * <ol>
+     *   <li>Appends a reserved suffix to cluster-state role mapping names to avoid conflicts with native role mappings.</li>
+     *   <li>Marks the metadata of cluster-state role mappings with a reserved read-only flag.</li>
+     *   <li>Removes internal metadata flag used in processing (see {@link RoleMappingMetadata#METADATA_NAME_FIELD}).</li>
+     * </ol>
+     */
+    private ExpressionRoleMapping translateClusterStateMapping(ExpressionRoleMapping mapping) {
+        Map<String, Object> metadata = new HashMap<>(mapping.getMetadata());
+        if (metadata.put(ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_METADATA_FLAG, true) != null) {
+            logger.error(
+                "Metadata field [{}] is reserved and will be overwritten with an internal system value. "
+                    + "Rename this field in your role mapping configuration.",
+                ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_METADATA_FLAG
+            );
+        }
+        metadata.remove(RoleMappingMetadata.METADATA_NAME_FIELD);
+        return new ExpressionRoleMapping(
+            ExpressionRoleMapping.addReadOnlySuffix(mapping.getName()),
+            mapping.getExpression(),
+            mapping.getRoles(),
+            mapping.getRoleTemplates(),
+            metadata,
+            mapping.isEnabled()
+        );
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java
index 82a3b4f000064..682ade925d2ec 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
@@ -16,24 +17,41 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
+import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 
+import static org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping.validateNoReadOnlySuffix;
+
 public class TransportPutRoleMappingAction extends HandledTransportAction<PutRoleMappingRequest, PutRoleMappingResponse> {
 
     private final NativeRoleMappingStore roleMappingStore;
+    private final ClusterStateRoleMapper clusterStateRoleMapper;
 
     @Inject
     public TransportPutRoleMappingAction(
         ActionFilters actionFilters,
         TransportService transportService,
-        NativeRoleMappingStore roleMappingStore
+        NativeRoleMappingStore roleMappingStore,
+        ClusterStateRoleMapper clusterStateRoleMapper
     ) {
         super(PutRoleMappingAction.NAME, transportService, actionFilters, PutRoleMappingRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.roleMappingStore = roleMappingStore;
+        this.clusterStateRoleMapper = clusterStateRoleMapper;
     }
 
     @Override
     protected void doExecute(Task task, final PutRoleMappingRequest request, final ActionListener<PutRoleMappingResponse> listener) {
+        validateNoReadOnlySuffix(request.getName());
+        if (clusterStateRoleMapper.hasMapping(request.getName())) {
+            // Allow to define a mapping with the same name in the native role mapping store as the file_settings namespace, but add a
+            // warning header to signal to the caller that this could be a problem.
+            HeaderWarning.addWarning(
+                "A read-only role mapping with the same name ["
+                    + request.getName()
+                    + "] has been previously defined in a configuration file. "
+                    + "Both role mappings will be used to determine role assignments."
+            );
+        }
         roleMappingStore.putRoleMapping(
             request,
             ActionListener.wrap(created -> listener.onResponse(new PutRoleMappingResponse(created)), listener::onFailure)
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index 90566e25b4ea5..03558e72fdca3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -139,7 +139,6 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE;
-import static org.elasticsearch.TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING;
 import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY;
@@ -148,6 +147,7 @@
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.WORKFLOWS_RESTRICTION_VERSION;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS;
@@ -2254,7 +2254,7 @@ public void queryApiKeys(SearchRequest searchRequest, boolean withLimitedBy, Act
                     TransportSearchAction.TYPE,
                     searchRequest,
                     ActionListener.wrap(searchResponse -> {
-                        long total = searchResponse.getHits().getTotalHits().value;
+                        long total = searchResponse.getHits().getTotalHits().value();
                         if (total == 0) {
                             logger.debug("No api keys found for query [{}]", searchRequest.source().query());
                             listener.onResponse(QueryApiKeysResult.EMPTY);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
index d866bd2a9d229..74a9aa7291ba4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
@@ -179,7 +179,7 @@ public void queryUsers(SearchRequest searchRequest, ActionListener<QueryUserResu
                     TransportSearchAction.TYPE,
                     searchRequest,
                     ActionListener.wrap(searchResponse -> {
-                        final long total = searchResponse.getHits().getTotalHits().value;
+                        final long total = searchResponse.getHits().getTotalHits().value();
                         if (total == 0) {
                             logger.debug("No users found for query [{}]", searchRequest.source().query());
                             listener.onResponse(QueryUserResults.EMPTY);
@@ -214,7 +214,7 @@ void getUserCount(final ActionListener<Long> listener) {
                         .setSize(0)
                         .setTrackTotalHits(true)
                         .request(),
-                    listener.<SearchResponse>safeMap(response -> response.getHits().getTotalHits().value),
+                    listener.<SearchResponse>safeMap(response -> response.getHits().getTotalHits().value()),
                     client::search
                 )
             );
@@ -706,7 +706,7 @@ void getAllReservedUserInfo(ActionListener<Map<String, ReservedUserInfo>> listen
                         @Override
                         public void onResponse(SearchResponse searchResponse) {
                             Map<String, ReservedUserInfo> userInfos = new HashMap<>();
-                            assert searchResponse.getHits().getTotalHits().value <= 10
+                            assert searchResponse.getHits().getTotalHits().value() <= 10
                                 : "there are more than 10 reserved users we need to change this to retrieve them all!";
                             for (SearchHit searchHit : searchResponse.getHits().getHits()) {
                                 Map<String, Object> sourceMap = searchHit.getSourceAsMap();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java
index 5dea6a938263c..99e3311283920 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
@@ -21,6 +22,7 @@
 
 import java.util.Objects;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.core.security.SecurityExtension.SecurityComponents;
 
@@ -28,8 +30,7 @@
  * A role mapper the reads the role mapping rules (i.e. {@link ExpressionRoleMapping}s) from the cluster state
  * (i.e. {@link RoleMappingMetadata}). This is not enabled by default.
  */
-public final class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCache implements ClusterStateListener {
-
+public class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCache implements ClusterStateListener {
     /**
      * This setting is never registered by the xpack security plugin - in order to disable the
      * cluster-state based role mapper another plugin must register it as a boolean setting
@@ -81,13 +82,26 @@ public void clusterChanged(ClusterChangedEvent event) {
         }
     }
 
-    private Set<ExpressionRoleMapping> getMappings() {
+    public boolean hasMapping(String name) {
+        if (enabled == false) {
+            return false;
+        }
+        return false == getMappings(Set.of(name)).isEmpty();
+    }
+
+    public Set<ExpressionRoleMapping> getMappings() {
+        return getMappings(null);
+    }
+
+    public Set<ExpressionRoleMapping> getMappings(@Nullable Set<String> names) {
         if (enabled == false) {
             return Set.of();
-        } else {
-            final Set<ExpressionRoleMapping> mappings = RoleMappingMetadata.getFromClusterState(clusterService.state()).getRoleMappings();
-            logger.trace("Retrieved [{}] mapping(s) from cluster state", mappings.size());
+        }
+        final Set<ExpressionRoleMapping> mappings = RoleMappingMetadata.getFromClusterState(clusterService.state()).getRoleMappings();
+        logger.trace("Retrieved [{}] mapping(s) from cluster state", mappings.size());
+        if (names == null || names.isEmpty()) {
             return mappings;
         }
+        return mappings.stream().filter(roleMapping -> names.contains(roleMapping.getName())).collect(Collectors.toSet());
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index 1c773a6e3963f..fa6187798da25 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -85,6 +85,7 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.NamedClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
+import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.support.StringMatcher;
 import org.elasticsearch.xpack.core.sql.SqlAsyncActionNames;
 import org.elasticsearch.xpack.security.action.user.TransportChangePasswordAction;
@@ -550,7 +551,7 @@ public void validateIndexPermissionsAreSubset(
                 Automaton existingPermissions = permissionMap.computeIfAbsent(entry.getKey(), role::allowedActionsMatcher);
                 for (String alias : entry.getValue()) {
                     Automaton newNamePermissions = permissionMap.computeIfAbsent(alias, role::allowedActionsMatcher);
-                    if (Operations.subsetOf(newNamePermissions, existingPermissions) == false) {
+                    if (Automatons.subsetOf(newNamePermissions, existingPermissions) == false) {
                         listener.onResponse(AuthorizationResult.deny());
                         return;
                     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java
index 40cb3ea4d9864..8ff535f3f6231 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java
@@ -10,7 +10,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -21,6 +20,7 @@
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.support.StringMatcher;
 
 import java.time.ZoneOffset;
@@ -195,7 +195,7 @@ private void logDeprecatedPermission(RoleDescriptor roleDescriptor) {
                         index.getName(),
                         i -> IndexPrivilege.get(indexPrivileges).getAutomaton()
                     );
-                    if (false == Operations.subsetOf(indexPrivilegeAutomaton, aliasPrivilegeAutomaton)) {
+                    if (false == Automatons.subsetOf(indexPrivilegeAutomaton, aliasPrivilegeAutomaton)) {
                         inferiorIndexNames.add(index.getName());
                     }
                 } else {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index 9ddda193dba39..4ae17a679d205 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -81,7 +81,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
 import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING;
@@ -91,6 +90,8 @@
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ROLE_TYPE;
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.SECURITY_ROLE_DESCRIPTION;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS;
 import static org.elasticsearch.xpack.security.support.SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION;
@@ -278,7 +279,7 @@ public void queryRoleDescriptors(SearchSourceBuilder searchSourceBuilder, Action
                     TransportSearchAction.TYPE,
                     searchRequest,
                     ActionListener.wrap(searchResponse -> {
-                        long total = searchResponse.getHits().getTotalHits().value;
+                        long total = searchResponse.getHits().getTotalHits().value();
                         if (total == 0) {
                             logger.debug("No roles found for query [{}]", searchRequest.source().query());
                             listener.onResponse(QueryRoleResult.EMPTY);
@@ -468,24 +469,25 @@ private Exception validateRoleDescriptor(RoleDescriptor role) {
             } else if (role.hasRemoteClusterPermissions()
                 && clusterService.state().getMinTransportVersion().before(ROLE_REMOTE_CLUSTER_PRIVS)) {
                     return new IllegalStateException(
-                        "all nodes must have version [" + ROLE_REMOTE_CLUSTER_PRIVS + "] or higher to support remote cluster privileges"
+                        "all nodes must have version ["
+                            + ROLE_REMOTE_CLUSTER_PRIVS.toReleaseVersion()
+                            + "] or higher to support remote cluster privileges"
                     );
-                } else if (role.hasDescription()
-                    && clusterService.state().getMinTransportVersion().before(TransportVersions.SECURITY_ROLE_DESCRIPTION)) {
+                } else if (role.hasDescription() && clusterService.state().getMinTransportVersion().before(SECURITY_ROLE_DESCRIPTION)) {
+                    return new IllegalStateException(
+                        "all nodes must have version ["
+                            + SECURITY_ROLE_DESCRIPTION.toReleaseVersion()
+                            + "] or higher to support specifying role description"
+                    );
+                } else if (Arrays.stream(role.getConditionalClusterPrivileges())
+                    .anyMatch(privilege -> privilege instanceof ConfigurableClusterPrivileges.ManageRolesPrivilege)
+                    && clusterService.state().getMinTransportVersion().before(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE)) {
                         return new IllegalStateException(
                             "all nodes must have version ["
-                                + TransportVersions.SECURITY_ROLE_DESCRIPTION.toReleaseVersion()
-                                + "] or higher to support specifying role description"
+                                + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion()
+                                + "] or higher to support the manage roles privilege"
                         );
-                    } else if (Arrays.stream(role.getConditionalClusterPrivileges())
-                        .anyMatch(privilege -> privilege instanceof ConfigurableClusterPrivileges.ManageRolesPrivilege)
-                        && clusterService.state().getMinTransportVersion().before(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE)) {
-                            return new IllegalStateException(
-                                "all nodes must have version ["
-                                    + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion()
-                                    + "] or higher to support the manage roles privilege"
-                            );
-                        }
+                    }
         try {
             DLSRoleQueryValidator.validateQueryField(role.getIndicesPrivileges(), xContentRegistry);
         } catch (ElasticsearchException | IllegalArgumentException e) {
@@ -645,8 +647,7 @@ XContentBuilder createRoleXContentBuilder(RoleDescriptor role) throws IOExceptio
             && clusterService.state().getMinTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS)) {
             builder.array(RoleDescriptor.Fields.REMOTE_CLUSTER.getPreferredName(), RemoteClusterPermissions.NONE);
         }
-        if (role.hasDescription() == false
-            && clusterService.state().getMinTransportVersion().onOrAfter(TransportVersions.SECURITY_ROLE_DESCRIPTION)) {
+        if (role.hasDescription() == false && clusterService.state().getMinTransportVersion().onOrAfter(SECURITY_ROLE_DESCRIPTION)) {
             builder.field(RoleDescriptor.Fields.DESCRIPTION.getPreferredName(), "");
         }
 
@@ -730,28 +731,28 @@ public void onResponse(MultiSearchResponse items) {
                             if (responses[0].isFailure()) {
                                 usageStats.put("size", 0);
                             } else {
-                                usageStats.put("size", responses[0].getResponse().getHits().getTotalHits().value);
+                                usageStats.put("size", responses[0].getResponse().getHits().getTotalHits().value());
                             }
                             if (responses[1].isFailure()) {
                                 usageStats.put("fls", false);
                             } else {
-                                usageStats.put("fls", responses[1].getResponse().getHits().getTotalHits().value > 0L);
+                                usageStats.put("fls", responses[1].getResponse().getHits().getTotalHits().value() > 0L);
                             }
 
                             if (responses[2].isFailure()) {
                                 usageStats.put("dls", false);
                             } else {
-                                usageStats.put("dls", responses[2].getResponse().getHits().getTotalHits().value > 0L);
+                                usageStats.put("dls", responses[2].getResponse().getHits().getTotalHits().value() > 0L);
                             }
                             if (responses[3].isFailure()) {
                                 usageStats.put("remote_indices", 0);
                             } else {
-                                usageStats.put("remote_indices", responses[3].getResponse().getHits().getTotalHits().value);
+                                usageStats.put("remote_indices", responses[3].getResponse().getHits().getTotalHits().value());
                             }
                             if (responses[4].isFailure()) {
                                 usageStats.put("remote_cluster", 0);
                             } else {
-                                usageStats.put("remote_cluster", responses[4].getResponse().getHits().getTotalHits().value);
+                                usageStats.put("remote_cluster", responses[4].getResponse().getHits().getTotalHits().value());
                             }
                             delegate.onResponse(usageStats);
                         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java
index b347ceb833f64..b347c278aae08 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java
@@ -414,19 +414,19 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                             logger.debug("error on counting total profiles", items[0].getFailure());
                             usage.put("total", 0L);
                         } else {
-                            usage.put("total", items[0].getResponse().getHits().getTotalHits().value);
+                            usage.put("total", items[0].getResponse().getHits().getTotalHits().value());
                         }
                         if (items[1].isFailure()) {
                             logger.debug("error on counting enabled profiles", items[0].getFailure());
                             usage.put("enabled", 0L);
                         } else {
-                            usage.put("enabled", items[1].getResponse().getHits().getTotalHits().value);
+                            usage.put("enabled", items[1].getResponse().getHits().getTotalHits().value());
                         }
                         if (items[2].isFailure()) {
                             logger.debug("error on counting recent profiles", items[0].getFailure());
                             usage.put("recent", 0L);
                         } else {
-                            usage.put("recent", items[2].getResponse().getHits().getTotalHits().value);
+                            usage.put("recent", items[2].getResponse().getHits().getTotalHits().value());
                         }
                         listener.onResponse(usage);
                     }, listener::onFailure)
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java
index 680f6b0f14f5e..ec287b01d0df2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -41,9 +40,7 @@ public RestAuthenticateAction(Settings settings, SecurityContext securityContext
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_security/_authenticate").replaces(GET, "/_xpack/security/_authenticate", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_security/_authenticate"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java
index 5615e2bc7915c..ec32b19f7e666 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java
@@ -8,9 +8,7 @@
 package org.elasticsearch.xpack.security.rest.action.apikey;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -66,7 +64,7 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
         try (XContentParser parser = request.contentParser()) {
-            final InvalidateApiKeyRequest invalidateApiKeyRequest = getObjectParser(request).parse(parser, null);
+            final InvalidateApiKeyRequest invalidateApiKeyRequest = PARSER.parse(parser, null);
             return channel -> client.execute(
                 InvalidateApiKeyAction.INSTANCE,
                 invalidateApiKeyRequest,
@@ -86,41 +84,6 @@ public String getName() {
         return "xpack_security_invalidate_api_key";
     }
 
-    private static ConstructingObjectParser<InvalidateApiKeyRequest, Void> getObjectParser(RestRequest request) {
-        if (request.getRestApiVersion() == RestApiVersion.V_7) {
-            final ConstructingObjectParser<InvalidateApiKeyRequest, Void> objectParser = new ConstructingObjectParser<>(
-                "invalidate_api_key_v7",
-                a -> {
-                    final String id = (String) a[5];
-                    @SuppressWarnings("unchecked")
-                    final List<String> ids = (List<String>) a[4];
-                    if (id != null && ids != null) {
-                        throw new IllegalArgumentException("Must use either [id] or [ids], not both at the same time");
-                    }
-                    final String[] idsArray;
-                    if (Strings.hasText(id)) {
-                        idsArray = new String[] { id };
-                    } else if (ids != null) {
-                        idsArray = ids.toArray(String[]::new);
-                    } else {
-                        idsArray = null;
-                    }
-                    return new InvalidateApiKeyRequest(
-                        (String) a[0],
-                        (String) a[1],
-                        (String) a[2],
-                        (a[3] == null) ? false : (Boolean) a[3],
-                        idsArray
-                    );
-                }
-            );
-            initObjectParser(objectParser, true);
-            return objectParser;
-        } else {
-            return PARSER;
-        }
-    }
-
     private static void initObjectParser(ConstructingObjectParser<InvalidateApiKeyRequest, Void> objectParser, boolean restCompatMode) {
         objectParser.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("realm_name"));
         objectParser.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("username"));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java
index 3102ba4e4ee36..fc07fa88d6d3d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -85,9 +84,7 @@ public RestGetTokenAction(Settings settings, XPackLicenseState xPackLicenseState
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/oauth2/token").replaces(POST, "/_xpack/security/oauth2/token", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_security/oauth2/token"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java
index d26d9c0e63aab..1959f8914d5e6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestInvalidateTokenAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequestFilter;
@@ -70,9 +69,7 @@ public RestInvalidateTokenAction(Settings settings, XPackLicenseState xPackLicen
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, "/_security/oauth2/token").replaces(DELETE, "/_xpack/security/oauth2/token", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(DELETE, "/_security/oauth2/token"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
index 6e0183c066668..967a2aec3f608 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -41,11 +40,7 @@ public RestDeletePrivilegesAction(Settings settings, XPackLicenseState licenseSt
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, "/_security/privilege/{application}/{privilege}")
-                .replaces(DELETE, "/_xpack/security/privilege/{application}/{privilege}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, "/_security/privilege/{application}/{privilege}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
index 827d0db186310..f8b8db44142ba 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -47,13 +46,9 @@ public RestGetPrivilegesAction(Settings settings, XPackLicenseState licenseState
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, "/_security/privilege/").replaces(GET, "/_xpack/security/privilege/", RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_security/privilege/{application}")
-                .replaces(GET, "/_xpack/security/privilege/{application}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, "/_security/privilege/{application}/{privilege}")
-                .replaces(GET, "/_xpack/security/privilege/{application}/{privilege}", RestApiVersion.V_7)
-                .build()
+            new Route(GET, "/_security/privilege/"),
+            new Route(GET, "/_security/privilege/{application}"),
+            new Route(GET, "/_security/privilege/{application}/{privilege}")
         );
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
index 53db1d135c125..78fe41e939316 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -44,10 +43,7 @@ public RestPutPrivilegesAction(Settings settings, XPackLicenseState licenseState
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(PUT, "/_security/privilege/").replaces(PUT, "/_xpack/security/privilege/", RestApiVersion.V_7).build(),
-            Route.builder(POST, "/_security/privilege/").replaces(POST, "/_xpack/security/privilege/", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(PUT, "/_security/privilege/"), new Route(POST, "/_security/privilege/"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestDisableProfileAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestDisableProfileAction.java
index 60e2cc332d3e9..08632c9c04759 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestDisableProfileAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestDisableProfileAction.java
@@ -33,10 +33,7 @@ public RestDisableProfileAction(Settings settings, XPackLicenseState licenseStat
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/profile/{uid}/_disable").build(),
-            Route.builder(PUT, "/_security/profile/{uid}/_disable").build()
-        );
+        return List.of(new Route(POST, "/_security/profile/{uid}/_disable"), new Route(PUT, "/_security/profile/{uid}/_disable"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestEnableProfileAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestEnableProfileAction.java
index 21cc647b3ef4b..74bc9d43002db 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestEnableProfileAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/profile/RestEnableProfileAction.java
@@ -33,10 +33,7 @@ public RestEnableProfileAction(Settings settings, XPackLicenseState licenseState
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/profile/{uid}/_enable").build(),
-            Route.builder(PUT, "/_security/profile/{uid}/_enable").build()
-        );
+        return List.of(new Route(POST, "/_security/profile/{uid}/_enable"), new Route(PUT, "/_security/profile/{uid}/_enable"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java
index 3b274bcea38d7..194029225da2d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -31,11 +30,7 @@ public RestClearRealmCacheAction(Settings settings, XPackLicenseState licenseSta
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/realm/{realms}/_clear_cache")
-                .replaces(POST, "/_xpack/security/realm/{realms}/_clear_cache", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, "/_security/realm/{realms}/_clear_cache"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkDeleteRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkDeleteRolesAction.java
index 683faf5cfa914..a420196b7237b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkDeleteRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkDeleteRolesAction.java
@@ -43,7 +43,7 @@ public RestBulkDeleteRolesAction(Settings settings, XPackLicenseState licenseSta
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(DELETE, "/_security/role").build());
+        return List.of(new Route(DELETE, "/_security/role"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkPutRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkPutRolesAction.java
index f132da09c4ec0..550e94b477034 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkPutRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestBulkPutRolesAction.java
@@ -34,7 +34,7 @@ public RestBulkPutRolesAction(Settings settings, XPackLicenseState licenseState,
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(POST, "/_security/role").build());
+        return List.of(new Route(POST, "/_security/role"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java
index e562d187764e6..546da4a1f31b1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -31,11 +30,7 @@ public RestClearRolesCacheAction(Settings settings, XPackLicenseState licenseSta
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/role/{name}/_clear_cache")
-                .replaces(POST, "/_xpack/security/role/{name}/_clear_cache", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, "/_security/role/{name}/_clear_cache"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java
index 0c3f0364e60d4..b74ef7eaa3190 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -37,9 +36,7 @@ public RestDeleteRoleAction(Settings settings, XPackLicenseState licenseState) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, "/_security/role/{name}").replaces(DELETE, "/_xpack/security/role/{name}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(DELETE, "/_security/role/{name}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java
index dc9ecbbc63a8d..82fe3dc5e12c2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -39,10 +38,7 @@ public RestGetRolesAction(Settings settings, XPackLicenseState licenseState) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_security/role/").replaces(GET, "/_xpack/security/role/", RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_security/role/{name}").replaces(GET, "/_xpack/security/role/{name}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_security/role/"), new Route(GET, "/_security/role/{name}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
index 6a819c098e9f1..a56ab4500f849 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -42,10 +41,7 @@ public RestPutRoleAction(Settings settings, XPackLicenseState licenseState, PutR
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/role/{name}").replaces(POST, "/_xpack/security/role/{name}", RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_security/role/{name}").replaces(PUT, "/_xpack/security/role/{name}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_security/role/{name}"), new Route(PUT, "/_security/role/{name}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java
index 5964228009c4b..eb4dca0546254 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -37,11 +36,7 @@ public RestDeleteRoleMappingAction(Settings settings, XPackLicenseState licenseS
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, "/_security/role_mapping/{name}")
-                .replaces(DELETE, "/_xpack/security/role_mapping/{name}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, "/_security/role_mapping/{name}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java
index 7a3378d843bca..21b20ffa551bd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -38,12 +37,7 @@ public RestGetRoleMappingsAction(Settings settings, XPackLicenseState licenseSta
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_security/role_mapping/").replaces(GET, "/_xpack/security/role_mapping/", RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_security/role_mapping/{name}")
-                .replaces(GET, "/_xpack/security/role_mapping/{name}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, "/_security/role_mapping/"), new Route(GET, "/_security/role_mapping/{name}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java
index 55562c8ee0138..378307022d8d8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -43,14 +42,7 @@ public RestPutRoleMappingAction(Settings settings, XPackLicenseState licenseStat
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/role_mapping/{name}")
-                .replaces(POST, "/_xpack/security/role_mapping/{name}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_security/role_mapping/{name}")
-                .replaces(PUT, "/_xpack/security/role_mapping/{name}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, "/_security/role_mapping/{name}"), new Route(PUT, "/_security/role_mapping/{name}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java
index 09527b251b6e8..f59d6984478ee 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequestFilter;
@@ -73,11 +72,7 @@ public RestSamlAuthenticateAction(Settings settings, XPackLicenseState licenseSt
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/saml/authenticate")
-                .replaces(POST, "/_xpack/security/saml/authenticate", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, "/_security/saml/authenticate"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlInvalidateSessionAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlInvalidateSessionAction.java
index f05bdb9a174dc..f326b1371209e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlInvalidateSessionAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlInvalidateSessionAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -53,9 +52,7 @@ public RestSamlInvalidateSessionAction(Settings settings, XPackLicenseState lice
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/saml/invalidate").replaces(POST, "/_xpack/security/saml/invalidate", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_security/saml/invalidate"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlLogoutAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlLogoutAction.java
index bc271e38d82b4..51e61d62af955 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlLogoutAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlLogoutAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -51,9 +50,7 @@ public RestSamlLogoutAction(Settings settings, XPackLicenseState licenseState) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/saml/logout").replaces(POST, "/_xpack/security/saml/logout", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_security/saml/logout"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlPrepareAuthenticationAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlPrepareAuthenticationAction.java
index cf3ad0d2edd3a..0903338578bac 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlPrepareAuthenticationAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlPrepareAuthenticationAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -55,9 +54,7 @@ public RestSamlPrepareAuthenticationAction(Settings settings, XPackLicenseState
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/saml/prepare").replaces(POST, "/_xpack/security/saml/prepare", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_security/saml/prepare"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestGetSecuritySettingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestGetSecuritySettingsAction.java
index 0b4ced0a20444..44b66a69b8f2a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestGetSecuritySettingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestGetSecuritySettingsAction.java
@@ -19,6 +19,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
 public class RestGetSecuritySettingsAction extends SecurityBaseRestHandler {
 
     public RestGetSecuritySettingsAction(Settings settings, XPackLicenseState licenseState) {
@@ -32,7 +34,7 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(RestRequest.Method.GET, "/_security/settings").build());
+        return List.of(new Route(GET, "/_security/settings"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestUpdateSecuritySettingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestUpdateSecuritySettingsAction.java
index 27ed6d2475d2c..f54b524df9ea1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestUpdateSecuritySettingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/settings/RestUpdateSecuritySettingsAction.java
@@ -19,6 +19,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
 public class RestUpdateSecuritySettingsAction extends SecurityBaseRestHandler {
 
     public RestUpdateSecuritySettingsAction(Settings settings, XPackLicenseState licenseState) {
@@ -32,7 +34,7 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(RestRequest.Method.PUT, "/_security/settings").build());
+        return List.of(new Route(PUT, "/_security/settings"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
index 68500c4d07e26..7840c34dbe393 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequestFilter;
@@ -48,14 +47,10 @@ public RestChangePasswordAction(Settings settings, SecurityContext securityConte
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(PUT, "/_security/user/{username}/_password")
-                .replaces(PUT, "/_xpack/security/user/{username}/_password", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/_security/user/{username}/_password")
-                .replaces(POST, "/_xpack/security/user/{username}/_password", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_security/user/_password").replaces(PUT, "/_xpack/security/user/_password", RestApiVersion.V_7).build(),
-            Route.builder(POST, "/_security/user/_password").replaces(POST, "/_xpack/security/user/_password", RestApiVersion.V_7).build()
+            new Route(PUT, "/_security/user/{username}/_password"),
+            new Route(POST, "/_security/user/{username}/_password"),
+            new Route(PUT, "/_security/user/_password"),
+            new Route(POST, "/_security/user/_password")
         );
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java
index 55c68b3b4b3f1..01136bdc21025 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -37,11 +36,7 @@ public RestDeleteUserAction(Settings settings, XPackLicenseState licenseState) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, "/_security/user/{username}")
-                .replaces(DELETE, "/_xpack/security/user/{username}", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(DELETE, "/_security/user/{username}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java
index 96e8ffd74a314..bbf5e0ca71842 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -48,9 +47,7 @@ public RestGetUserPrivilegesAction(Settings settings, SecurityContext securityCo
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_security/user/_privileges").replaces(GET, "/_xpack/security/user/_privileges", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_security/user/_privileges"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
index ebc956add641a..7a24854ef6d9c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -39,10 +38,7 @@ public RestGetUsersAction(Settings settings, XPackLicenseState licenseState) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_security/user/").replaces(GET, "/_xpack/security/user/", RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_security/user/{username}").replaces(GET, "/_xpack/security/user/{username}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_security/user/"), new Route(GET, "/_security/user/{username}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
index a896d4855b73d..f2233a7e19fd0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
@@ -60,18 +59,10 @@ public RestHasPrivilegesAction(
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(GET, "/_security/user/{username}/_has_privileges")
-                .replaces(GET, "/_xpack/security/user/{username}/_has_privileges", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/_security/user/{username}/_has_privileges")
-                .replaces(POST, "/_xpack/security/user/{username}/_has_privileges", RestApiVersion.V_7)
-                .build(),
-            Route.builder(GET, "/_security/user/_has_privileges")
-                .replaces(GET, "/_xpack/security/user/_has_privileges", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/_security/user/_has_privileges")
-                .replaces(POST, "/_xpack/security/user/_has_privileges", RestApiVersion.V_7)
-                .build()
+            new Route(GET, "/_security/user/{username}/_has_privileges"),
+            new Route(POST, "/_security/user/{username}/_has_privileges"),
+            new Route(GET, "/_security/user/_has_privileges"),
+            new Route(POST, "/_security/user/_has_privileges")
         );
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestProfileHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestProfileHasPrivilegesAction.java
index 77c35438d0d0b..a2de75b8b4f64 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestProfileHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestProfileHasPrivilegesAction.java
@@ -38,10 +38,7 @@ public RestProfileHasPrivilegesAction(Settings settings, XPackLicenseState licen
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_security/profile/_has_privileges").build(),
-            Route.builder(POST, "/_security/profile/_has_privileges").build()
-        );
+        return List.of(new Route(GET, "/_security/profile/_has_privileges"), new Route(POST, "/_security/profile/_has_privileges"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
index 0d20103f06b57..bbaa8ac915127 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequestFilter;
@@ -45,12 +44,7 @@ public RestPutUserAction(Settings settings, XPackLicenseState licenseState) {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_security/user/{username}")
-                .replaces(POST, "/_xpack/security/user/{username}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_security/user/{username}").replaces(PUT, "/_xpack/security/user/{username}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_security/user/{username}"), new Route(PUT, "/_security/user/{username}"));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java
index f34450cbbe1ef..1ee42b191f31a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -40,18 +39,10 @@ public RestSetEnabledAction(Settings settings, XPackLicenseState licenseState) {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(POST, "/_security/user/{username}/_enable")
-                .replaces(POST, "/_xpack/security/user/{username}/_enable", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_security/user/{username}/_enable")
-                .replaces(PUT, "/_xpack/security/user/{username}/_enable", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/_security/user/{username}/_disable")
-                .replaces(POST, "/_xpack/security/user/{username}/_disable", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_security/user/{username}/_disable")
-                .replaces(PUT, "/_xpack/security/user/{username}/_disable", RestApiVersion.V_7)
-                .build()
+            new Route(POST, "/_security/user/{username}/_enable"),
+            new Route(PUT, "/_security/user/{username}/_enable"),
+            new Route(POST, "/_security/user/{username}/_disable"),
+            new Route(PUT, "/_security/user/{username}/_disable")
         );
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
index 5ec76a8dc3d01..5cd8cba763d3d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
@@ -77,8 +77,8 @@ public void migrate(SecurityIndexManager indexManager, Client client, ActionList
 
                 client.search(countRequest, ActionListener.wrap(response -> {
                     // If there are no roles, skip migration
-                    if (response.getHits().getTotalHits().value > 0) {
-                        logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value + "] roles");
+                    if (response.getHits().getTotalHits().value() > 0) {
+                        logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value() + "] roles");
                         updateRolesByQuery(indexManager, client, filterQuery, listener);
                     } else {
                         listener.onResponse(null);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheckTests.java
deleted file mode 100644
index 85e8d6dd38125..0000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheckTests.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.security;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.BootstrapCheck;
-import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.env.BuildVersion;
-import org.elasticsearch.env.NodeMetadata;
-import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.license.ClusterStateLicenseService;
-import org.elasticsearch.license.License;
-import org.elasticsearch.license.LicensesMetadata;
-import org.elasticsearch.license.TestUtils;
-import org.elasticsearch.license.internal.TrialLicenseVersion;
-import org.elasticsearch.test.AbstractBootstrapCheckTestCase;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.xpack.core.XPackSettings;
-
-import static org.hamcrest.Matchers.is;
-import static org.mockito.Mockito.mock;
-
-public class SecurityImplicitBehaviorBootstrapCheckTests extends AbstractBootstrapCheckTestCase {
-
-    @UpdateForV9(owner = UpdateForV9.Owner.SECURITY)
-    @AwaitsFix(bugUrl = "requires updates for version 9.0 bump")
-    public void testUpgradeFrom8xWithImplicitSecuritySettings() throws Exception {
-        final BuildVersion previousVersion = toBuildVersion(VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, null));
-        NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, IndexVersion.current());
-        nodeMetadata = nodeMetadata.upgradeToCurrentVersion();
-        ClusterStateLicenseService licenseService = mock(ClusterStateLicenseService.class);
-        BootstrapCheck.BootstrapCheckResult result = new SecurityImplicitBehaviorBootstrapCheck(nodeMetadata, licenseService).check(
-            createTestContext(
-                Settings.EMPTY,
-                createLicensesMetadata(TrialLicenseVersion.fromXContent(previousVersion.toString()), randomFrom("basic", "trial"))
-            )
-        );
-        assertThat(result.isSuccess(), is(true));
-    }
-
-    @UpdateForV9(owner = UpdateForV9.Owner.SECURITY)
-    @AwaitsFix(bugUrl = "requires updates for version 9.0 bump")
-    public void testUpgradeFrom8xWithExplicitSecuritySettings() throws Exception {
-        final BuildVersion previousVersion = toBuildVersion(VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, null));
-        NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, IndexVersion.current());
-        nodeMetadata = nodeMetadata.upgradeToCurrentVersion();
-        ClusterStateLicenseService licenseService = mock(ClusterStateLicenseService.class);
-        BootstrapCheck.BootstrapCheckResult result = new SecurityImplicitBehaviorBootstrapCheck(nodeMetadata, licenseService).check(
-            createTestContext(
-                Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build(),
-                createLicensesMetadata(TrialLicenseVersion.fromXContent(previousVersion.toString()), randomFrom("basic", "trial"))
-            )
-        );
-        assertThat(result.isSuccess(), is(true));
-    }
-
-    private Metadata createLicensesMetadata(TrialLicenseVersion era, String licenseMode) throws Exception {
-        License license = TestUtils.generateSignedLicense(licenseMode, TimeValue.timeValueHours(2));
-        return Metadata.builder().putCustom(LicensesMetadata.TYPE, new LicensesMetadata(license, era)).build();
-    }
-
-    private static BuildVersion toBuildVersion(Version version) {
-        return BuildVersion.fromVersionId(version.id());
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java
index 6e8698f095d32..010c19e8cc1b1 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
@@ -19,21 +18,26 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
-import org.hamcrest.Matchers;
 import org.junit.Before;
 
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
-import static org.hamcrest.Matchers.arrayContaining;
+import static org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_METADATA_FLAG;
+import static org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anySet;
 import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -42,8 +46,10 @@ public class TransportGetRoleMappingsActionTests extends ESTestCase {
 
     private NativeRoleMappingStore store;
     private TransportGetRoleMappingsAction action;
-    private AtomicReference<Set<String>> namesRef;
-    private List<ExpressionRoleMapping> result;
+    private AtomicReference<Set<String>> nativeNamesRef;
+    private AtomicReference<Set<String>> clusterStateNamesRef;
+    private List<ExpressionRoleMapping> nativeMappings;
+    private Set<ExpressionRoleMapping> clusterStateMappings;
 
     @SuppressWarnings("unchecked")
     @Before
@@ -58,68 +64,219 @@ public void setupMocks() {
             null,
             Collections.emptySet()
         );
-        action = new TransportGetRoleMappingsAction(mock(ActionFilters.class), transportService, store);
+        ClusterStateRoleMapper clusterStateRoleMapper = mock();
+        action = new TransportGetRoleMappingsAction(mock(ActionFilters.class), transportService, store, clusterStateRoleMapper);
 
-        namesRef = new AtomicReference<>(null);
-        result = Collections.emptyList();
+        nativeNamesRef = new AtomicReference<>(null);
+        clusterStateNamesRef = new AtomicReference<>(null);
+        nativeMappings = Collections.emptyList();
+        clusterStateMappings = Collections.emptySet();
+
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            assert args.length == 1;
+            clusterStateNamesRef.set((Set<String>) args[0]);
+            return clusterStateMappings;
+        }).when(clusterStateRoleMapper).getMappings(anySet());
 
         doAnswer(invocation -> {
             Object[] args = invocation.getArguments();
             assert args.length == 2;
-            namesRef.set((Set<String>) args[0]);
+            nativeNamesRef.set((Set<String>) args[0]);
             ActionListener<List<ExpressionRoleMapping>> listener = (ActionListener<List<ExpressionRoleMapping>>) args[1];
-            listener.onResponse(result);
+            listener.onResponse(nativeMappings);
             return null;
         }).when(store).getRoleMappings(nullable(Set.class), any(ActionListener.class));
     }
 
-    public void testGetSingleRole() throws Exception {
-        final PlainActionFuture<GetRoleMappingsResponse> future = new PlainActionFuture<>();
-        final GetRoleMappingsRequest request = new GetRoleMappingsRequest();
-        request.setNames("everyone");
+    public void testGetSingleRoleMappingNativeOnly() throws Exception {
+        testGetMappings(List.of(mapping("everyone")), Collections.emptySet(), Set.of("everyone"), Set.of("everyone"), "everyone");
+    }
 
-        final ExpressionRoleMapping mapping = mock(ExpressionRoleMapping.class);
-        result = Collections.singletonList(mapping);
-        action.doExecute(mock(Task.class), request, future);
-        assertThat(future.get(), notNullValue());
-        assertThat(future.get().mappings(), arrayContaining(mapping));
-        assertThat(namesRef.get(), containsInAnyOrder("everyone"));
+    public void testGetMultipleNamedRoleMappingsNativeOnly() throws Exception {
+        testGetMappings(
+            List.of(mapping("admin"), mapping("engineering"), mapping("sales"), mapping("finance")),
+            Collections.emptySet(),
+            Set.of("admin", "engineering", "sales", "finance"),
+            Set.of("admin", "engineering", "sales", "finance"),
+            "admin",
+            "engineering",
+            "sales",
+            "finance"
+        );
     }
 
-    public void testGetMultipleNamedRoles() throws Exception {
-        final PlainActionFuture<GetRoleMappingsResponse> future = new PlainActionFuture<>();
-        final GetRoleMappingsRequest request = new GetRoleMappingsRequest();
-        request.setNames("admin", "engineering", "sales", "finance");
+    public void testGetAllRoleMappingsNativeOnly() throws Exception {
+        testGetMappings(
+            List.of(mapping("admin"), mapping("engineering"), mapping("sales"), mapping("finance")),
+            Collections.emptySet(),
+            Set.of(),
+            Set.of()
+        );
+    }
 
-        final ExpressionRoleMapping mapping1 = mock(ExpressionRoleMapping.class);
-        final ExpressionRoleMapping mapping2 = mock(ExpressionRoleMapping.class);
-        final ExpressionRoleMapping mapping3 = mock(ExpressionRoleMapping.class);
-        result = Arrays.asList(mapping1, mapping2, mapping3);
+    public void testGetSingleRoleMappingClusterStateOnly() throws Exception {
+        testGetMappings(List.of(), Set.of(mapping("everyone")), Set.of("everyone"), Set.of("everyone"), "everyone");
+    }
 
-        action.doExecute(mock(Task.class), request, future);
+    public void testGetMultipleNamedRoleMappingsClusterStateOnly() throws Exception {
+        testGetMappings(
+            List.of(),
+            Set.of(mapping("admin"), mapping("engineering"), mapping("sales"), mapping("finance")),
+            Set.of("admin", "engineering", "sales", "finance"),
+            Set.of("admin", "engineering", "sales", "finance"),
+            "admin",
+            "engineering",
+            "sales",
+            "finance"
+        );
+    }
+
+    public void testGetAllRoleMappingsClusterStateOnly() throws Exception {
+        testGetMappings(
+            List.of(),
+            Set.of(mapping("admin"), mapping("engineering"), mapping("sales"), mapping("finance")),
+            Set.of(),
+            Set.of()
+        );
+    }
+
+    public void testGetSingleRoleMappingBoth() throws Exception {
+        testGetMappings(List.of(mapping("everyone")), Set.of(mapping("everyone")), Set.of("everyone"), Set.of("everyone"), "everyone");
+    }
+
+    public void testGetMultipleNamedRoleMappingsBoth() throws Exception {
+        testGetMappings(
+            List.of(mapping("admin"), mapping("engineering")),
+            Set.of(mapping("sales"), mapping("finance")),
+            Set.of("admin", "engineering", "sales", "finance"),
+            Set.of("admin", "engineering", "sales", "finance"),
+            "admin",
+            "engineering",
+            "sales",
+            "finance"
+        );
+    }
+
+    public void testGetAllRoleMappingsClusterBoth() throws Exception {
+        testGetMappings(List.of(mapping("admin"), mapping("engineering")), Set.of(mapping("admin"), mapping("sales")), Set.of(), Set.of());
+    }
+
+    public void testGetSingleRoleMappingQueryWithReadOnlySuffix() throws Exception {
+        testGetMappings(
+            List.of(),
+            Set.of(mapping("everyone")),
+            // suffix not stripped for native store query
+            Set.of("everyone" + READ_ONLY_ROLE_MAPPING_SUFFIX),
+            // suffix is stripped for cluster state store
+            Set.of("everyone"),
+            "everyone" + READ_ONLY_ROLE_MAPPING_SUFFIX
+        );
+
+        testGetMappings(
+            List.of(),
+            Set.of(mapping("everyoneread-only-operator-mapping")),
+            Set.of(
+                "everyoneread-only-operator-mapping",
+                "everyone-read-only-operator-mapping-",
+                "everyone-read-only-operator-mapping-more"
+            ),
+            // suffix that is similar but not the same is not stripped
+            Set.of(
+                "everyoneread-only-operator-mapping",
+                "everyone-read-only-operator-mapping-",
+                "everyone-read-only-operator-mapping-more"
+            ),
+            "everyoneread-only-operator-mapping",
+            "everyone-read-only-operator-mapping-",
+            "everyone-read-only-operator-mapping-more"
+        );
+
+        testGetMappings(
+            List.of(mapping("everyone")),
+            Set.of(mapping("everyone")),
+            // suffix not stripped for native store query
+            Set.of("everyone" + READ_ONLY_ROLE_MAPPING_SUFFIX, "everyone"),
+            // suffix is stripped for cluster state store
+            Set.of("everyone"),
+            "everyone" + READ_ONLY_ROLE_MAPPING_SUFFIX,
+            "everyone"
+        );
+    }
+
+    public void testClusterStateRoleMappingWithFallbackNameOmitted() throws ExecutionException, InterruptedException {
+        testGetMappings(
+            List.of(),
+            Set.of(mapping("name_not_available_after_deserialization")),
+            Set.of(),
+            Set.of("name_not_available_after_deserialization"),
+            Set.of("name_not_available_after_deserialization"),
+            "name_not_available_after_deserialization"
+        );
 
-        final GetRoleMappingsResponse response = future.get();
-        assertThat(response, notNullValue());
-        assertThat(response.mappings(), arrayContainingInAnyOrder(mapping1, mapping2, mapping3));
-        assertThat(namesRef.get(), containsInAnyOrder("admin", "engineering", "sales", "finance"));
+        testGetMappings(
+            List.of(mapping("name_not_available_after_deserialization")),
+            Set.of(mapping("name_not_available_after_deserialization")),
+            Set.of(),
+            Set.of("name_not_available_after_deserialization"),
+            Set.of("name_not_available_after_deserialization"),
+            "name_not_available_after_deserialization"
+        );
+    }
+
+    private void testGetMappings(
+        List<ExpressionRoleMapping> returnedNativeMappings,
+        Set<ExpressionRoleMapping> returnedClusterStateMappings,
+        Set<String> expectedNativeNames,
+        Set<String> expectedClusterStateNames,
+        String... names
+    ) throws InterruptedException, ExecutionException {
+        testGetMappings(
+            returnedNativeMappings,
+            returnedClusterStateMappings,
+            returnedClusterStateMappings.stream().map(this::expectedClusterStateMapping).collect(Collectors.toSet()),
+            expectedNativeNames,
+            expectedClusterStateNames,
+            names
+        );
     }
 
-    public void testGetAllRoles() throws Exception {
+    private void testGetMappings(
+        List<ExpressionRoleMapping> returnedNativeMappings,
+        Set<ExpressionRoleMapping> returnedClusterStateMappings,
+        Set<ExpressionRoleMapping> expectedClusterStateMappings,
+        Set<String> expectedNativeNames,
+        Set<String> expectedClusterStateNames,
+        String... names
+    ) throws InterruptedException, ExecutionException {
         final PlainActionFuture<GetRoleMappingsResponse> future = new PlainActionFuture<>();
         final GetRoleMappingsRequest request = new GetRoleMappingsRequest();
-        request.setNames(Strings.EMPTY_ARRAY);
-
-        final ExpressionRoleMapping mapping1 = mock(ExpressionRoleMapping.class);
-        final ExpressionRoleMapping mapping2 = mock(ExpressionRoleMapping.class);
-        final ExpressionRoleMapping mapping3 = mock(ExpressionRoleMapping.class);
-        result = Arrays.asList(mapping1, mapping2, mapping3);
+        request.setNames(names);
 
+        nativeMappings = returnedNativeMappings;
+        clusterStateMappings = returnedClusterStateMappings;
         action.doExecute(mock(Task.class), request, future);
+        assertThat(future.get(), notNullValue());
+        List<ExpressionRoleMapping> combined = new ArrayList<>(returnedNativeMappings);
+        combined.addAll(expectedClusterStateMappings);
+        ExpressionRoleMapping[] actualMappings = future.get().mappings();
+        assertThat(actualMappings, arrayContainingInAnyOrder(combined.toArray(new ExpressionRoleMapping[0])));
+        assertThat(nativeNamesRef.get(), containsInAnyOrder(expectedNativeNames.toArray(new String[0])));
+        assertThat(clusterStateNamesRef.get(), containsInAnyOrder(expectedClusterStateNames.toArray(new String[0])));
+    }
 
-        final GetRoleMappingsResponse response = future.get();
-        assertThat(response, notNullValue());
-        assertThat(response.mappings(), arrayContainingInAnyOrder(mapping1, mapping2, mapping3));
-        assertThat(namesRef.get(), Matchers.nullValue(Set.class));
+    private ExpressionRoleMapping mapping(String name) {
+        return new ExpressionRoleMapping(name, null, null, null, Map.of(), true);
     }
 
+    private ExpressionRoleMapping expectedClusterStateMapping(ExpressionRoleMapping mapping) {
+        return new ExpressionRoleMapping(
+            mapping.getName() + READ_ONLY_ROLE_MAPPING_SUFFIX,
+            null,
+            null,
+            null,
+            Map.of(READ_ONLY_ROLE_MAPPING_METADATA_FLAG, true),
+            true
+        );
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java
index 6f789a10a3a6c..6d1ac864d20fd 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
+import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.junit.Before;
 
@@ -29,18 +30,21 @@
 
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.iterableWithSize;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TransportPutRoleMappingActionTests extends ESTestCase {
 
     private NativeRoleMappingStore store;
     private TransportPutRoleMappingAction action;
     private AtomicReference<PutRoleMappingRequest> requestRef;
+    private ClusterStateRoleMapper clusterStateRoleMapper;
 
     @SuppressWarnings("unchecked")
     @Before
@@ -55,7 +59,9 @@ public void setupMocks() {
             null,
             Collections.emptySet()
         );
-        action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, store);
+        clusterStateRoleMapper = mock();
+        when(clusterStateRoleMapper.hasMapping(any())).thenReturn(false);
+        action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, store, clusterStateRoleMapper);
 
         requestRef = new AtomicReference<>(null);
 
@@ -85,6 +91,41 @@ public void testPutValidMapping() throws Exception {
         assertThat(mapping.getMetadata().get("dumb"), equalTo(true));
     }
 
+    public void testValidMappingClashingClusterStateMapping() throws Exception {
+        final FieldExpression expression = new FieldExpression("username", Collections.singletonList(new FieldExpression.FieldValue("*")));
+        final PutRoleMappingResponse response = put("anarchy", expression, "superuser", Collections.singletonMap("dumb", true));
+        when(clusterStateRoleMapper.hasMapping(any())).thenReturn(true);
+
+        assertThat(response.isCreated(), equalTo(true));
+
+        final ExpressionRoleMapping mapping = requestRef.get().getMapping();
+        assertThat(mapping.getExpression(), is(expression));
+        assertThat(mapping.isEnabled(), equalTo(true));
+        assertThat(mapping.getName(), equalTo("anarchy"));
+        assertThat(mapping.getRoles(), iterableWithSize(1));
+        assertThat(mapping.getRoles(), contains("superuser"));
+        assertThat(mapping.getMetadata(), aMapWithSize(1));
+        assertThat(mapping.getMetadata().get("dumb"), equalTo(true));
+    }
+
+    public void testInvalidSuffix() {
+        final FieldExpression expression = new FieldExpression("username", Collections.singletonList(new FieldExpression.FieldValue("*")));
+        String name = ExpressionRoleMapping.addReadOnlySuffix("anarchy");
+        final var ex = expectThrows(IllegalArgumentException.class, () -> {
+            put(name, expression, "superuser", Collections.singletonMap("dumb", true));
+        });
+        assertThat(
+            ex.getMessage(),
+            containsString(
+                "Invalid mapping name ["
+                    + name
+                    + "]. ["
+                    + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX
+                    + "] is not an allowed suffix"
+            )
+        );
+    }
+
     private PutRoleMappingResponse put(String name, FieldExpression expression, String role, Map<String, Object> metadata)
         throws Exception {
         final PutRoleMappingRequest request = new PutRoleMappingRequest();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
index fa6eb307933ec..996291c52c71f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
@@ -158,7 +158,6 @@
 import java.util.stream.IntStream;
 import java.util.stream.LongStream;
 
-import static org.elasticsearch.TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.elasticsearch.test.ActionListenerUtils.anyActionListener;
@@ -170,6 +169,7 @@
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_METADATA_KEY;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_TYPE_KEY;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.WORKFLOWS_RESTRICTION_VERSION;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR;
 import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7;
 import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptorTests.java
index 2c4a03b7df501..b09527061f0d5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptorTests.java
@@ -91,7 +91,7 @@ public void testRequestCacheWillBeDisabledWhenSearchRemoteIndices() {
             0,
             3,
             String[]::new,
-            () -> randomAlphaOfLengthBetween(0, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
+            () -> randomAlphaOfLengthBetween(1, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
         );
         final ArrayList<String> allIndices = Arrays.stream(ArrayUtils.concat(localIndices, remoteIndices))
             .collect(Collectors.toCollection(ArrayList::new));
@@ -121,7 +121,7 @@ public void testHasRemoteIndices() {
             0,
             3,
             String[]::new,
-            () -> randomAlphaOfLengthBetween(0, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
+            () -> randomAlphaOfLengthBetween(1, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
         );
         final ArrayList<String> allIndices = Arrays.stream(ArrayUtils.concat(localIndices, remoteIndices))
             .collect(Collectors.toCollection(ArrayList::new));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java
index 0a2c40d2a257a..a4d9dacd1a63d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.automaton.MinimizationOperations;
+import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -137,8 +137,8 @@ public void testParseFile() throws Exception {
         assertThat(group.indices().length, is(1));
         assertThat(group.indices()[0], equalTo("idx3"));
         assertThat(group.privilege(), notNullValue());
-        assertTrue(Operations.subsetOf(IndexPrivilege.READ.getAutomaton(), group.privilege().getAutomaton()));
-        assertTrue(Operations.subsetOf(IndexPrivilege.WRITE.getAutomaton(), group.privilege().getAutomaton()));
+        assertTrue(AutomatonTestUtil.subsetOf(IndexPrivilege.READ.getAutomaton(), group.privilege().getAutomaton()));
+        assertTrue(AutomatonTestUtil.subsetOf(IndexPrivilege.WRITE.getAutomaton(), group.privilege().getAutomaton()));
 
         descriptor = roles.get("role1.ab");
         assertNotNull(descriptor);
@@ -181,9 +181,9 @@ public void testParseFile() throws Exception {
         assertThat(group.indices()[0], equalTo("/.*_.*/"));
         assertThat(group.privilege(), notNullValue());
         assertTrue(
-            Operations.sameLanguage(
+            AutomatonTestUtil.sameLanguage(
                 group.privilege().getAutomaton(),
-                MinimizationOperations.minimize(
+                Operations.determinize(
                     Operations.union(IndexPrivilege.READ.getAutomaton(), IndexPrivilege.WRITE.getAutomaton()),
                     Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
                 )
@@ -236,7 +236,7 @@ public void testParseFile() throws Exception {
         assertThat(group.indices().length, is(1));
         assertThat(group.indices()[0], equalTo("field_idx"));
         assertThat(group.privilege(), notNullValue());
-        assertTrue(Operations.sameLanguage(group.privilege().getAutomaton(), IndexPrivilege.READ.getAutomaton()));
+        assertTrue(AutomatonTestUtil.sameLanguage(group.privilege().getAutomaton(), IndexPrivilege.READ.getAutomaton()));
         assertTrue(group.getFieldPermissions().grantsAccessTo("foo"));
         assertTrue(group.getFieldPermissions().grantsAccessTo("boo"));
         assertTrue(group.getFieldPermissions().hasFieldLevelSecurity());
@@ -258,7 +258,7 @@ public void testParseFile() throws Exception {
         assertThat(group.indices().length, is(1));
         assertThat(group.indices()[0], equalTo("query_idx"));
         assertThat(group.privilege(), notNullValue());
-        assertTrue(Operations.sameLanguage(group.privilege().getAutomaton(), IndexPrivilege.READ.getAutomaton()));
+        assertTrue(AutomatonTestUtil.sameLanguage(group.privilege().getAutomaton(), IndexPrivilege.READ.getAutomaton()));
         assertFalse(group.getFieldPermissions().hasFieldLevelSecurity());
         assertThat(group.getQuery(), notNullValue());
 
@@ -279,7 +279,7 @@ public void testParseFile() throws Exception {
         assertThat(group.indices().length, is(1));
         assertThat(group.indices()[0], equalTo("query_fields_idx"));
         assertThat(group.privilege(), notNullValue());
-        assertTrue(Operations.sameLanguage(group.privilege().getAutomaton(), IndexPrivilege.READ.getAutomaton()));
+        assertTrue(AutomatonTestUtil.sameLanguage(group.privilege().getAutomaton(), IndexPrivilege.READ.getAutomaton()));
         assertTrue(group.getFieldPermissions().grantsAccessTo("foo"));
         assertTrue(group.getFieldPermissions().grantsAccessTo("boo"));
         assertTrue(group.getFieldPermissions().hasFieldLevelSecurity());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
index bfa358d0b7d6e..2b8a77d63588a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
@@ -98,6 +98,7 @@
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomClusterPrivileges;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomRemoteIndicesPrivileges;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomRoleDescriptorMetadata;
+import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.contains;
@@ -488,7 +489,7 @@ enum TEST_MODE {
             switch (testMode) {
                 case REMOTE_CLUSTER_PRIVS -> {
                     transportVersionBeforeAdvancedRemoteClusterSecurity = TransportVersionUtils.getPreviousVersion(
-                        TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS
+                        ROLE_REMOTE_CLUSTER_PRIVS
                     );
                     remoteIndicesPrivileges = null;
                 }
@@ -549,7 +550,7 @@ enum TEST_MODE {
                 containsString(
                     "all nodes must have version ["
                         + (TEST_MODE.REMOTE_CLUSTER_PRIVS.equals(testMode)
-                            ? TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS
+                            ? ROLE_REMOTE_CLUSTER_PRIVS.toReleaseVersion()
                             : TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion())
                         + "] or higher to support remote "
                         + (remoteIndicesPrivileges != null ? "indices" : "cluster")
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java
index 3d3f96b98d5e5..d294fb50046d6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java
@@ -158,7 +158,7 @@ public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(
                 }
             }
         );
-        DiscoveryNode remoteNode = remoteTransportService.getLocalDiscoNode();
+        DiscoveryNode remoteNode = remoteTransportService.getLocalNode();
         remoteTransportService.registerRequestHandler(
             RemoteClusterNodesAction.TYPE.name(),
             EsExecutors.DIRECT_EXECUTOR_SERVICE,
diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java
index d12d093dd5b8d..ee7438dfca428 100644
--- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java
+++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java
@@ -327,11 +327,8 @@ public void testAutoExpandDuringRestart() throws Exception {
         ensureGreen("myindex");
 
         putNodeShutdown(primaryNodeId, SingleNodeShutdownMetadata.Type.RESTART, null);
-        // registering node shutdown entry does not perform reroute, neither should it.
-        // we provoke it here in the test to ensure that auto-expansion has run.
-        updateIndexSettings(Settings.builder().put("index.routing.allocation.exclude.name", "non-existent"), "myindex");
 
-        assertBusy(() -> assertIndexSetting("myindex", "index.number_of_replicas", "1"));
+        assertIndexSetting("myindex", "index.number_of_replicas", "1");
         indexRandomData("myindex");
 
         internalCluster().restartNode(primaryNode, new InternalTestCluster.RestartCallback() {
@@ -361,9 +358,6 @@ public void testAutoExpandDuringReplace() throws Exception {
         var replacementNodeName = "node_t2";
 
         putNodeShutdown(nodeIdToReplace, SingleNodeShutdownMetadata.Type.REPLACE, replacementNodeName);
-        // registering node shutdown entry does not perform reroute, neither should it.
-        // we provoke it here in the test to ensure that auto-expansion has run.
-        updateIndexSettings(Settings.builder().put("index.routing.allocation.exclude.name", "non-existent"), "index");
 
         ensureGreen("index");
         assertIndexSetting("index", "index.number_of_replicas", "1");
@@ -381,6 +375,32 @@ public void testAutoExpandDuringReplace() throws Exception {
         assertIndexSetting("index", "index.number_of_replicas", "1");
     }
 
+    public void testAutoExpandDuringShutdown() throws Exception {
+
+        var node1 = internalCluster().startNode();
+        var node2 = internalCluster().startNode();
+
+        createIndex("index", indexSettings(1, 0).put("index.auto_expand_replicas", randomFrom("0-all", "0-1")).build());
+        indexRandomData("index");
+
+        ensureGreen("index");
+        assertIndexSetting("index", "index.number_of_replicas", "1");
+
+        var nodeNameToShutdown = randomFrom(node1, node2);
+        var nodeIdToShutdown = getNodeId(nodeNameToShutdown);
+
+        putNodeShutdown(nodeIdToShutdown, SingleNodeShutdownMetadata.Type.REMOVE, null);
+
+        ensureGreen("index");
+        assertIndexSetting("index", "index.number_of_replicas", "0");
+
+        assertBusy(() -> assertNodeShutdownStatus(nodeIdToShutdown, COMPLETE));
+        internalCluster().stopNode(nodeIdToShutdown);
+
+        ensureGreen("index");
+        assertIndexSetting("index", "index.number_of_replicas", "0");
+    }
+
     public void testNodeShutdownWithUnassignedShards() throws Exception {
         final String nodeA = internalCluster().startNode();
         final String nodeAId = getNodeId(nodeA);
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java
index 810bd8f6e9ceb..95fd97cd5931f 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java
@@ -122,7 +122,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                         metadata.getAllocationDelay().getStringRep()
                     );
                 }
-                builder.timeField(
+                builder.timestampFieldsFromUnixEpochMillis(
                     SingleNodeShutdownMetadata.STARTED_AT_MILLIS_FIELD.getPreferredName(),
                     SingleNodeShutdownMetadata.STARTED_AT_READABLE_FIELD,
                     metadata.getStartedAtMillis()
@@ -138,7 +138,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                     builder.field(TARGET_NODE_NAME_FIELD.getPreferredName(), metadata.getTargetNodeName());
                 }
                 if (metadata.getGracePeriod() != null) {
-                    builder.timeField(
+                    builder.timestampField(
                         SingleNodeShutdownMetadata.GRACE_PERIOD_FIELD.getPreferredName(),
                         metadata.getGracePeriod().getStringRep()
                     );
diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java
index d42d45e430627..e5171a7c51650 100644
--- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java
+++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java
@@ -281,7 +281,7 @@ public void testRetentionWhileSnapshotInProgress() throws Exception {
                             completedSnapshotName,
                             Strings.arrayToCommaDelimitedString(resp.getHits().getHits())
                         );
-                        assertThat(resp.getHits().getTotalHits().value, equalTo(2L));
+                        assertThat(resp.getHits().getTotalHits().value(), equalTo(2L));
                     }
                 );
             });
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java
index 60fdba2051041..8426ad491e353 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java
@@ -220,7 +220,7 @@ public final void writeTo(StreamOutput out) throws IOException {
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         {
-            builder.timeField(TIMESTAMP.getPreferredName(), "timestamp_string", timestamp);
+            builder.timestampFieldsFromUnixEpochMillis(TIMESTAMP.getPreferredName(), "timestamp_string", timestamp);
             builder.field(POLICY_ID.getPreferredName(), policyId);
             builder.field(REPOSITORY.getPreferredName(), repository);
             builder.field(SNAPSHOT_NAME.getPreferredName(), snapshotName);
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
index 0fcc4b8007c6d..b993633e3d17d 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.reservedstate.service.ReservedClusterStateService;
 import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask;
 import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor;
+import org.elasticsearch.reservedstate.service.ReservedStateVersionCheck;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockUtils;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -399,7 +400,7 @@ public void testOperatorControllerFromJSONContent() throws IOException {
         AtomicReference<Exception> x = new AtomicReference<>();
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) {
-            controller.process("operator", parser, x::set);
+            controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), x::set);
 
             assertThat(x.get(), instanceOf(IllegalStateException.class));
             assertThat(x.get().getMessage(), containsString("Error processing state change request for operator"));
@@ -419,7 +420,7 @@ public void testOperatorControllerFromJSONContent() throws IOException {
         );
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, testJSON)) {
-            controller.process("operator", parser, Assert::assertNull);
+            controller.process("operator", parser, randomFrom(ReservedStateVersionCheck.values()), Assert::assertNull);
         }
     }
 
diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java
index df8dc54bb7490..405a9926e2e5f 100644
--- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java
+++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java
@@ -1595,7 +1595,7 @@ private void assertDocumentsAreEqual(String indexName, int docCount) {
                     int docIdToMatch = randomIntBetween(0, docCount - 1);
                     assertResponse(searchRequestBuilder.setQuery(QueryBuilders.termQuery("field", docIdToMatch)), searchResponse -> {
                         assertThat(searchResponse.getSuccessfulShards(), equalTo(1));
-                        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+                        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L));
                         SearchHit searchHit = searchResponse.getHits().getAt(0);
                         Map<String, Object> source = searchHit.getSourceAsMap();
                         assertThat(source, is(notNullValue()));
@@ -1613,7 +1613,7 @@ private void assertDocumentsAreEqual(String indexName, int docCount) {
 
     private void assertSearchResponseContainsAllIndexedDocs(SearchResponse searchResponse, long docCount) {
         assertThat(searchResponse.getSuccessfulShards(), equalTo(1));
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(docCount));
+        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(docCount));
         for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
             SearchHit searchHit = searchResponse.getHits().getAt(i);
             Map<String, Object> source = searchHit.getSourceAsMap();
diff --git a/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java b/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java
index b082698254d17..6e7f2d82cfb1d 100644
--- a/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java
+++ b/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java
@@ -26,15 +26,12 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.IOUtils;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.StoreFileMetadata;
-import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.indices.recovery.plan.ShardRecoveryPlan;
 import org.elasticsearch.indices.recovery.plan.ShardSnapshot;
 import org.elasticsearch.indices.recovery.plan.ShardSnapshotsService;
@@ -63,7 +60,6 @@
 import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
 import static org.elasticsearch.index.engine.Engine.ES_VERSION;
 import static org.elasticsearch.index.engine.Engine.HISTORY_UUID_KEY;
-import static org.elasticsearch.test.index.IndexVersionUtils.randomVersionBetween;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -203,8 +199,6 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
         });
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    @AwaitsFix(bugUrl = "this is testing pre-7.0 upgrade behavior so probably needs some updating")
     public void testLogicallyEquivalentSnapshotIsUsedEvenIfFilesAreDifferent() throws Exception {
         createStore(store -> {
             boolean shareFilesWithSource = randomBoolean();
@@ -217,18 +211,8 @@ public void testLogicallyEquivalentSnapshotIsUsedEvenIfFilesAreDifferent() throw
             final IndexVersion snapshotVersion;
             final Version luceneVersion;
             if (compatibleVersion) {
-                snapshotVersion = randomBoolean() ? null : IndexVersionUtils.randomCompatibleVersion(random());
-                // If snapshotVersion is not present,
-                // then lucene version must be < RecoverySettings.SEQ_NO_SNAPSHOT_RECOVERIES_SUPPORTED_VERSION
-                if (snapshotVersion == null) {
-                    luceneVersion = randomVersionBetween(
-                        random(),
-                        IndexVersions.V_7_0_0,
-                        RecoverySettings.SNAPSHOT_RECOVERIES_SUPPORTED_INDEX_VERSION
-                    ).luceneVersion();
-                } else {
-                    luceneVersion = IndexVersionUtils.randomCompatibleVersion(random()).luceneVersion();
-                }
+                snapshotVersion = IndexVersionUtils.randomCompatibleVersion(random());
+                luceneVersion = snapshotVersion.luceneVersion();
             } else {
                 snapshotVersion = IndexVersion.fromId(Integer.MAX_VALUE);
                 luceneVersion = org.apache.lucene.util.Version.parse("255.255.255");
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java
index 90130811c1218..143d2671b9eab 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java
@@ -158,7 +158,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.timeField("timestamp_in_millis", "timestamp", timestampMillis);
+        builder.timestampFieldsFromUnixEpochMillis("timestamp_in_millis", "timestamp", timestampMillis);
 
         if (anomaly() != null) {
             builder.field("anomaly", anomaly());
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java
index 3c64d140e2b56..e7b9156d5fb66 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java
@@ -129,7 +129,7 @@ public void testKnownIssueWithCellLeftOfDatelineTouchingPolygonOnRightOfDateline
                     client().prepareSearch("test").setTrackTotalHits(true).setQuery(queryBuilder),
                     innerResponse -> assertThat(
                         "Bucket " + bucket.getKeyAsString(),
-                        innerResponse.getHits().getTotalHits().value,
+                        innerResponse.getHits().getTotalHits().value(),
                         Matchers.equalTo(bucket.getDocCount())
                     )
                 );
@@ -320,7 +320,7 @@ private void assertQuery(List<InternalGeoGridBucket> buckets, BiFunction<String,
                 client().prepareSearch("test").setTrackTotalHits(true).setQuery(queryBuilder),
                 response -> assertThat(
                     "Expected hits at precision " + precision + " for H3 cell " + bucket.getKeyAsString(),
-                    response.getHits().getTotalHits().value,
+                    response.getHits().getTotalHits().value(),
                     Matchers.equalTo(bucket.getDocCount())
                 )
             );
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java
index b4a3a07502abf..b4d7a472591bd 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java
@@ -175,7 +175,7 @@ public void testStorePolygonDateLine() throws Exception {
         indexRandom(true, prepareIndex("test").setId("0").setSource(source, XContentType.JSON));
 
         assertNoFailuresAndResponse(client().prepareSearch("test").setFetchSource(false).addStoredField("shape"), response -> {
-            assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
             SearchHit searchHit = response.getHits().getAt(0);
             assertThat(searchHit.field("shape").getValue(), instanceOf(BytesRef.class));
             BytesRef bytesRef = searchHit.field("shape").getValue();
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
index 1c013aba52261..4f23b6de4c37d 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
@@ -247,7 +247,7 @@ public void testFieldAlias() {
         assertResponse(
             client().prepareSearch(INDEX).setQuery(new ShapeQueryBuilder("alias", queryGeometry).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertTrue(response.getHits().getTotalHits().value > 0);
+                assertTrue(response.getHits().getTotalHits().value() > 0);
             }
         );
     }
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java
index 1ac6bf3b6fd31..e26066cd89c50 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java
@@ -112,7 +112,7 @@ public void testIndexPointsFilterRectangle() {
             client().prepareSearch(defaultIndexName)
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, rectangle).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
                 assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
@@ -123,7 +123,7 @@ public void testIndexPointsFilterRectangle() {
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName).setQuery(new ShapeQueryBuilder(defaultFieldName, rectangle)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
                 assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
@@ -138,7 +138,7 @@ public void testIndexPointsCircle() {
             client().prepareSearch(defaultIndexName)
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, circle).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(2L));
                 assertThat(response.getHits().getHits().length, equalTo(2));
                 assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
                 assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
@@ -154,7 +154,7 @@ public void testIndexPointsPolygon() {
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, polygon).relation(ShapeRelation.INTERSECTS)),
             response -> {
                 SearchHits searchHits = response.getHits();
-                assertThat(searchHits.getTotalHits().value, equalTo(2L));
+                assertThat(searchHits.getTotalHits().value(), equalTo(2L));
                 assertThat(searchHits.getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
                 assertThat(searchHits.getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
             }
@@ -175,7 +175,7 @@ public void testIndexPointsMultiPolygon() {
             client().prepareSearch(defaultIndexName)
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, mp).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(3L));
                 assertThat(response.getHits().getHits().length, equalTo(3));
                 assertThat(response.getHits().getAt(0).getId(), not(equalTo("3")));
                 assertThat(response.getHits().getAt(1).getId(), not(equalTo("3")));
@@ -191,7 +191,7 @@ public void testIndexPointsRectangle() {
             client().prepareSearch(defaultIndexName)
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, rectangle).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
             }
@@ -232,7 +232,7 @@ public void testIndexPointsIndexedRectangle() throws Exception {
                         .indexedShapePath(indexedShapePath)
                 ),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
                 assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
             }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java
index 66f5597be543e..2713afc149e05 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java
@@ -230,8 +230,8 @@ public void testGeoShapeQueryAcrossDateline() throws IOException {
 
             try (IndexReader reader = w.getReader()) {
                 IndexSearcher searcher = newSearcher(reader);
-                assertThat(searcher.search(sameShapeQuery, 1).totalHits.value, equalTo(1L));
-                assertThat(searcher.search(pointOnDatelineQuery, 1).totalHits.value, equalTo(1L));
+                assertThat(searcher.search(sameShapeQuery, 1).totalHits.value(), equalTo(1L));
+                assertThat(searcher.search(pointOnDatelineQuery, 1).totalHits.value(), equalTo(1L));
             }
         }
     }
@@ -261,8 +261,8 @@ public void testShapeQuery() throws IOException {
 
             try (IndexReader reader = w.getReader()) {
                 IndexSearcher searcher = newSearcher(reader);
-                assertThat(searcher.search(sameShapeQuery, 1).totalHits.value, equalTo(1L));
-                assertThat(searcher.search(centerPointQuery, 1).totalHits.value, equalTo(1L));
+                assertThat(searcher.search(sameShapeQuery, 1).totalHits.value(), equalTo(1L));
+                assertThat(searcher.search(centerPointQuery, 1).totalHits.value(), equalTo(1L));
             }
         }
     }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitorTests.java
index 8e2f713e6ed3e..d9cbbaafa1779 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitorTests.java
@@ -83,9 +83,10 @@ private void doTestGeometry(LongFunction<Geometry> h3ToGeometry, boolean hasArea
             visitor.reset(centerChild);
             reader.visit(visitor);
             if (hasArea) {
-                if (h3CrossesDateline && visitor.getLeftX() > visitor.getRightX()) {
-                    // if both polygons crosses the dateline it cannot be inside due to the polygon splitting technique
-                    assertEquals("failing h3: " + h3, GeoRelation.QUERY_CROSSES, visitor.relation());
+                if (h3CrossesDateline) {
+                    // if the h3 crosses the dateline, we might get CROSSES due to the polygon splitting technique. We can't
+                    // be sure which one is the correct one, so we just check that it is not DISJOINT
+                    assertNotSame("failing h3: " + h3, GeoRelation.QUERY_DISJOINT, visitor.relation());
                 } else {
                     assertEquals("failing h3: " + h3, GeoRelation.QUERY_INSIDE, visitor.relation());
                 }
diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java
index d8534b963c2d7..20fb342aa4b38 100644
--- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java
+++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java
@@ -1350,8 +1350,8 @@ private void setupDataForDateTimeTests(long randomLongDate, Long randomLongDateN
 
         indexSimpleDocumentWithBooleanValues("1", true, randomLongDate, randomLongDateNanos);
         index("test", "2", builder -> {
-            builder.timeField("test_date", null);
-            builder.timeField("test_date_nanos", null);
+            builder.timestampField("test_date", null);
+            builder.timestampField("test_date_nanos", null);
         });
     }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
index b2ce91140de76..411a4cda868f0 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
@@ -10,7 +10,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.search.ClosePointInTimeRequest;
@@ -101,7 +100,6 @@
 import static org.elasticsearch.action.ActionListener.wrap;
 import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.LENIENT;
 import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.NONE;
-import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_UNSIGNED_LONG;
 
 // TODO: add retry/back-off
 public class Querier {
@@ -202,7 +200,7 @@ public static void closePointInTime(Client client, BytesReference pointInTimeId,
     public static SearchRequest prepareRequest(SearchSourceBuilder source, SqlConfiguration cfg, boolean includeFrozen, String... indices) {
         source.timeout(cfg.requestTimeout());
 
-        SearchRequest searchRequest = new SearchRequest(Version.fromId(INTRODUCING_UNSIGNED_LONG.id));
+        SearchRequest searchRequest = new SearchRequest();
         if (source.pointInTimeBuilder() == null) {
             searchRequest.indices(indices);
             searchRequest.indicesOptions(
@@ -225,7 +223,7 @@ protected static void logSearchResponse(SearchResponse response, Logger logger)
         }
 
         var totalHits = response.getHits().getTotalHits();
-        var hits = totalHits != null ? "hits " + totalHits.relation + " " + totalHits.value + ", " : "";
+        var hits = totalHits != null ? "hits " + totalHits.relation() + " " + totalHits.value() + ", " : "";
         logger.trace(
             "Got search response [{}{} aggregations: [{}], {} failed shards, {} skipped shards, "
                 + "{} successful shards, {} total shards, took {}, timed out [{}]]",
@@ -550,7 +548,7 @@ protected List<BucketExtractor> initBucketExtractors(SearchResponse response) {
 
             List<BucketExtractor> exts = new ArrayList<>(refs.size());
             TotalHits totalHits = response.getHits().getTotalHits();
-            ConstantExtractor totalCount = new TotalHitsExtractor(totalHits == null ? -1L : totalHits.value);
+            ConstantExtractor totalCount = new TotalHitsExtractor(totalHits == null ? -1L : totalHits.value());
             for (QueryContainer.FieldInfo ref : refs) {
                 exts.add(createExtractor(ref.extraction(), totalCount));
             }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java
index 78976ea7e83c0..cf52a5f5d7126 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java
@@ -76,7 +76,7 @@ public Object extract(Bucket bucket) {
             throw new SqlIllegalArgumentException("Cannot find an aggregation named {}", name);
         }
 
-        if (agg.getHits().getTotalHits() == null || agg.getHits().getTotalHits().value == 0) {
+        if (agg.getHits().getTotalHits() == null || agg.getHits().getTotalHits().value() == 0) {
             return null;
         }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java
index 26b6bfdc0cead..bf68e174f033c 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlClearCursorAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.sql.plugin;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -19,7 +18,6 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xpack.sql.action.Protocol;
 import org.elasticsearch.xpack.sql.action.SqlClearCursorAction;
 import org.elasticsearch.xpack.sql.action.SqlClearCursorRequest;
 import org.elasticsearch.xpack.sql.action.SqlClearCursorResponse;
@@ -29,17 +27,14 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.sql.proto.CoreProtocol.CLEAR_CURSOR_REST_ENDPOINT;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestSqlClearCursorAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, Protocol.CLEAR_CURSOR_REST_ENDPOINT)
-                .replaces(POST, Protocol.CLEAR_CURSOR_DEPRECATED_REST_ENDPOINT, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, CLEAR_CURSOR_REST_ENDPOINT));
     }
 
     @Override
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
index 43051e9e16160..63f942714f8de 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.sql.plugin;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -17,7 +16,6 @@
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.sql.action.Protocol;
 import org.elasticsearch.xpack.sql.action.SqlQueryAction;
 import org.elasticsearch.xpack.sql.action.SqlQueryRequest;
 
@@ -29,6 +27,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.ql.util.LoggingUtils.logOnFailure;
+import static org.elasticsearch.xpack.sql.proto.CoreProtocol.SQL_QUERY_REST_ENDPOINT;
 import static org.elasticsearch.xpack.sql.proto.CoreProtocol.URL_PARAM_DELIMITER;
 
 @ServerlessScope(Scope.PUBLIC)
@@ -37,14 +36,7 @@ public class RestSqlQueryAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, Protocol.SQL_QUERY_REST_ENDPOINT)
-                .replaces(GET, Protocol.SQL_QUERY_DEPRECATED_REST_ENDPOINT, RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, Protocol.SQL_QUERY_REST_ENDPOINT)
-                .replaces(POST, Protocol.SQL_QUERY_DEPRECATED_REST_ENDPOINT, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, SQL_QUERY_REST_ENDPOINT), new Route(POST, SQL_QUERY_REST_ENDPOINT));
     }
 
     @Override
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlStatsAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlStatsAction.java
index 7ba50b2f5aaf6..897226f50d75c 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlStatsAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlStatsAction.java
@@ -8,25 +8,20 @@
 package org.elasticsearch.xpack.sql.plugin;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.xpack.sql.action.Protocol;
 
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.sql.proto.CoreProtocol.SQL_STATS_REST_ENDPOINT;
 
 public class RestSqlStatsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, Protocol.SQL_STATS_REST_ENDPOINT)
-                .replaces(GET, Protocol.SQL_STATS_DEPRECATED_REST_ENDPOINT, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, SQL_STATS_REST_ENDPOINT));
     }
 
     @Override
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlTranslateAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlTranslateAction.java
index 04565d77b68d2..690c340e30d44 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlTranslateAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlTranslateAction.java
@@ -7,14 +7,12 @@
 package org.elasticsearch.xpack.sql.plugin;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.sql.action.Protocol;
 import org.elasticsearch.xpack.sql.action.SqlTranslateAction;
 import org.elasticsearch.xpack.sql.action.SqlTranslateRequest;
 
@@ -23,6 +21,7 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.sql.proto.CoreProtocol.SQL_TRANSLATE_REST_ENDPOINT;
 
 /**
  * REST action for translating SQL queries into ES requests
@@ -32,14 +31,7 @@ public class RestSqlTranslateAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, Protocol.SQL_TRANSLATE_REST_ENDPOINT)
-                .replaces(GET, Protocol.SQL_TRANSLATE_DEPRECATED_REST_ENDPOINT, RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, Protocol.SQL_TRANSLATE_REST_ENDPOINT)
-                .replaces(POST, Protocol.SQL_TRANSLATE_DEPRECATED_REST_ENDPOINT, RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(GET, SQL_TRANSLATE_REST_ENDPOINT), new Route(POST, SQL_TRANSLATE_REST_ENDPOINT));
     }
 
     @Override
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
index 7a76ffe8eb109..41fa66ae36aeb 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
@@ -9,7 +9,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -57,7 +56,6 @@
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.ActionListener.wrap;
 import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
-import static org.elasticsearch.xpack.ql.plugin.TransportActionUtils.executeRequestWithRetryAttempt;
 import static org.elasticsearch.xpack.sql.plugin.Transports.clusterName;
 import static org.elasticsearch.xpack.sql.plugin.Transports.username;
 import static org.elasticsearch.xpack.sql.proto.Mode.CLI;
@@ -161,22 +159,11 @@ public static void operation(
         );
 
         if (Strings.hasText(request.cursor()) == false) {
-            executeRequestWithRetryAttempt(
-                clusterService,
-                listener::onFailure,
-                onFailure -> planExecutor.sql(
-                    cfg,
-                    request.query(),
-                    request.params(),
-                    wrap(p -> listener.onResponse(createResponseWithSchema(request, p, task)), onFailure)
-                ),
-                node -> transportService.sendRequest(
-                    node,
-                    SqlQueryAction.NAME,
-                    request,
-                    new ActionListenerResponseHandler<>(listener, SqlQueryResponse::new, EsExecutors.DIRECT_EXECUTOR_SERVICE)
-                ),
-                log
+            planExecutor.sql(
+                cfg,
+                request.query(),
+                request.params(),
+                wrap(p -> listener.onResponse(createResponseWithSchema(request, p, task)), listener::onFailure)
             );
         } else {
             Tuple<Cursor, ZoneId> decoded = Cursors.decodeFromStringWithZone(request.cursor(), planExecutor.writeableRegistry());
diff --git a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
index 556a417fb5e79..988ee93bda6b4 100644
--- a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
+++ b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
@@ -43,6 +43,7 @@ public class XPackRestIT extends AbstractXPackRestTest {
         .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB")
         .user("x_pack_rest_user", "x-pack-test-password")
         .feature(FeatureFlag.TIME_SERIES_MODE)
+        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
         .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
         .configFile("service_tokens", Resource.fromClasspath("service_tokens"))
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
index 4e8f82d507a5f..96145e84ad2cd 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
@@ -430,14 +430,14 @@ setup:
         - method: POST
           path: /_query
           parameters: [ ]
-          capabilities: [ named_parameter_for_field_and_function_names ]
+          capabilities: [ named_parameter_for_field_and_function_names_simplified_syntax ]
       reason: "named or positional parameters for field names"
 
   - do:
       esql.query:
         body:
           query: 'from test | stats x = count(?f1), y = sum(?f2) by ?f3 | sort ?f3 | keep ?f3, x, y  | limit 3'
-          params: [{"f1" : {"value" : "time", "kind" : "identifier" }}, {"f2" : { "value" : "count", "kind" : "identifier" }}, {"f3" : { "value" : "color", "kind" : "identifier" }}]
+          params: [{"f1" : {"identifier" : "time"}}, {"f2" : { "identifier" : "count" }}, {"f3" : { "identifier" : "color"}}]
 
   - length: {columns: 3}
   - match: {columns.0.name: "color"}
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml
index 33c9cc7558672..83234901ae8f2 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml
@@ -155,3 +155,19 @@ setup:
       esql.query:
         body:
           query: 'FROM test [metadata _source] | STATS COUNT_DISTINCT(_source)'
+
+---
+"sort on _source not allowed":
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [sorting_on_source_and_counters_forbidden]
+      reason: "Sorting on _source shouldn't have been possible"
+  - do:
+      catch: /cannot sort on _source/
+      esql.query:
+        body:
+          query: 'FROM test metadata _source | sort _source'
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
index 642407ac6d45b..ebf464ba667db 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
@@ -178,11 +178,19 @@ cast counter then filter:
 
 ---
 sort on counter without cast:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [sorting_on_source_and_counters_forbidden]
+      reason: "Sorting on counters shouldn't have been possible"
   - do:
-      catch: bad_request
+      catch: /cannot sort on counter_long/
       esql.query:
         body:
-          query: 'from test |  KEEP k8s.pod.network.tx | sort @k8s.pod.network.tx | limit 1'
+          query: 'from test |  KEEP k8s.pod.network.tx | sort k8s.pod.network.tx | limit 1'
 
 ---
 cast then sort on counter:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
index e100f30717aef..049895bc9f31a 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
@@ -504,3 +504,52 @@ double nested declared in mapping:
 
   # The `nested` field is not visible, nor are any of it's subfields.
   - match: { columns: [{name: name, type: keyword}] }
+
+---
+semantic_text declared in mapping:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ semantic_text_type ]
+      reason: "support for semantic_text type"
+  - do:
+      indices.create:
+        index: test_semantic_text
+        body:
+          settings:
+            number_of_shards: 5
+          mappings:
+            properties:
+              semantic_text_field:
+                type: semantic_text
+                inference_id: my_inference_id
+  - do:
+      bulk:
+        index: test_semantic_text
+        refresh: true
+        body:
+          - { "index": { } }
+          - {
+              "semantic_text_field": {
+                "text": "be excellent to each other",
+                "inference": {
+                  "inference_id": "my_inference_id",
+                  "model_settings": {
+                    "task_type": "sparse_embedding"
+                  },
+                  "chunks": [{ "text": "be excellent to each other", "embeddings": { "a": 1,"b": 2 } }]
+              }
+            }
+          }
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_semantic_text'
+  - match: { columns: [{name: semantic_text_field, type: semantic_text}] }
+  - length: { values: 1 }
+  - match: { values.0: ["be excellent to each other"] }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
index 8bbdb27a87d1a..b51bbdc4d2f87 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
@@ -5,7 +5,7 @@ setup:
         - method: POST
           path: /_query
           parameters: [ method, path, parameters, capabilities ]
-          capabilities: [ no_meta ]
+          capabilities: [ function_stats ]
       reason: "META command removed which changes the count of the data returned"
       test_runner_features: [capabilities]
 
@@ -23,7 +23,86 @@ setup:
                 type: integer
 
 ---
-"Basic ESQL usage output (telemetry)":
+"Basic ESQL usage output (telemetry) snapshot version":
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [ snapshot_test_for_telemetry ]
+      reason: "Test that should only be executed on snapshot versions"
+
+  - do: {xpack.usage: {}}
+  - match: { esql.available: true }
+  - match: { esql.enabled: true }
+  - length: { esql.features: 15 }
+  - set: {esql.features.dissect: dissect_counter}
+  - set: {esql.features.drop: drop_counter}
+  - set: {esql.features.eval: eval_counter}
+  - set: {esql.features.enrich: enrich_counter}
+  - set: {esql.features.from: from_counter}
+  - set: {esql.features.grok: grok_counter}
+  - set: {esql.features.keep: keep_counter}
+  - set: {esql.features.limit: limit_counter}
+  - set: {esql.features.mv_expand: mv_expand_counter}
+  - set: {esql.features.rename: rename_counter}
+  - set: {esql.features.row: row_counter}
+  - set: {esql.features.show: show_counter}
+  - set: {esql.features.sort: sort_counter}
+  - set: {esql.features.stats: stats_counter}
+  - set: {esql.features.where: where_counter}
+  - length: { esql.queries: 3 }
+  - set: {esql.queries.rest.total: rest_total_counter}
+  - set: {esql.queries.rest.failed: rest_failed_counter}
+  - set: {esql.queries.kibana.total: kibana_total_counter}
+  - set: {esql.queries.kibana.failed: kibana_failed_counter}
+  - set: {esql.queries._all.total: all_total_counter}
+  - set: {esql.queries._all.failed: all_failed_counter}
+  - set: {esql.functions.max: functions_max}
+  - set: {esql.functions.min: functions_min}
+  - set: {esql.functions.cos: functions_cos}
+  - set: {esql.functions.to_long: functions_to_long}
+  - set: {esql.functions.coalesce: functions_coalesce}
+
+  - do:
+      esql.query:
+        body:
+          query: 'from test | where data > 2 and to_long(data) > 2 | sort count desc | limit 5 | stats m = max(data)'
+
+  - do: {xpack.usage: {}}
+  - match: { esql.available: true }
+  - match: { esql.enabled: true }
+  - match: {esql.features.dissect: $dissect_counter}
+  - match: {esql.features.eval: $eval_counter}
+  - match: {esql.features.grok: $grok_counter}
+  - gt: {esql.features.limit: $limit_counter}
+  - gt: {esql.features.sort: $sort_counter}
+  - gt: {esql.features.stats: $stats_counter}
+  - gt: {esql.features.where: $where_counter}
+  - gt: {esql.queries.rest.total: $rest_total_counter}
+  - match: {esql.queries.rest.failed: $rest_failed_counter}
+  - match: {esql.queries.kibana.total: $kibana_total_counter}
+  - match: {esql.queries.kibana.failed: $kibana_failed_counter}
+  - gt: {esql.queries._all.total: $all_total_counter}
+  - match: {esql.queries._all.failed: $all_failed_counter}
+  - gt: {esql.functions.max: $functions_max}
+  - match: {esql.functions.min: $functions_min}
+  - match: {esql.functions.cos: $functions_cos}
+  - gt: {esql.functions.to_long: $functions_to_long}
+  - match: {esql.functions.coalesce: $functions_coalesce}
+  - length: {esql.functions: 117} # check the "sister" test below for a likely update to the same esql.functions length check
+
+---
+"Basic ESQL usage output (telemetry) non-snapshot version":
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [ non_snapshot_test_for_telemetry ]
+      reason: "Test that should only be executed on release versions"
 
   - do: {xpack.usage: {}}
   - match: { esql.available: true }
@@ -51,11 +130,16 @@ setup:
   - set: {esql.queries.kibana.failed: kibana_failed_counter}
   - set: {esql.queries._all.total: all_total_counter}
   - set: {esql.queries._all.failed: all_failed_counter}
+  - set: {esql.functions.max: functions_max}
+  - set: {esql.functions.min: functions_min}
+  - set: {esql.functions.cos: functions_cos}
+  - set: {esql.functions.to_long: functions_to_long}
+  - set: {esql.functions.coalesce: functions_coalesce}
 
   - do:
       esql.query:
         body:
-          query: 'from test | where data > 2 | sort count desc | limit 5 | stats m = max(data)'
+          query: 'from test | where data > 2 and to_long(data) > 2 | sort count desc | limit 5 | stats m = max(data)'
 
   - do: {xpack.usage: {}}
   - match: { esql.available: true }
@@ -73,3 +157,9 @@ setup:
   - match: {esql.queries.kibana.failed: $kibana_failed_counter}
   - gt: {esql.queries._all.total: $all_total_counter}
   - match: {esql.queries._all.failed: $all_failed_counter}
+  - gt: {esql.functions.max: $functions_max}
+  - match: {esql.functions.min: $functions_min}
+  - match: {esql.functions.cos: $functions_cos}
+  - gt: {esql.functions.to_long: $functions_to_long}
+  - match: {esql.functions.coalesce: $functions_coalesce}
+  - length: {esql.functions: 115} # check the "sister" test above for a likely update to the same esql.functions length check
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
index 939f153b8b0ea..88ef03a22d70c 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
@@ -563,7 +563,7 @@ setup:
   - match: { values.1.0: "Payroll Specialist" }
 
 ---
-values:
+"values function":
   - requires:
       cluster_features: esql.agg_values
       reason: "values is available in 8.14+"
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml
index 11be68cc764e2..cdc69001d33ef 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml
@@ -39,23 +39,4 @@
           }
   - match: { error.reason: "Unknown task_type [bad]" }
 
----
-"Test get all":
-  - do:
-      inference.get:
-        inference_id: "*"
-  - length: { endpoints: 1}
-  - match: { endpoints.0.inference_id: ".elser-2" }
-
-  - do:
-      inference.get:
-        inference_id: _all
-  - length: { endpoints: 1}
-  - match: { endpoints.0.inference_id: ".elser-2" }
-
-  - do:
-      inference.get:
-        inference_id: ""
-  - length: { endpoints: 1}
-  - match: { endpoints.0.inference_id: ".elser-2" }
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java
index 8ee23e38f9ffe..0ba29fef8e06d 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java
@@ -123,7 +123,7 @@ protected void masterOperation(
                     Arrays.toString(transformCountSuccess.getShardFailures())
                 );
             }
-            long totalTransforms = transformCountSuccess.getHits().getTotalHits().value;
+            long totalTransforms = transformCountSuccess.getHits().getTotalHits().value();
             if (totalTransforms == 0) {
                 var usage = new TransformFeatureSetUsage(transformsCountByState, Collections.emptyMap(), new TransformIndexerStats());
                 listener.onResponse(new XPackUsageFeatureResponse(usage));
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProvider.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProvider.java
index f49d5fc96f3ab..cd06a4cadaa37 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProvider.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProvider.java
@@ -84,7 +84,7 @@ public void sourceHasChanged(TransformCheckpoint lastCheckpoint, ActionListener<
             client,
             TransportSearchAction.TYPE,
             searchRequest,
-            ActionListener.wrap(r -> listener.onResponse(r.getHits().getTotalHits().value > 0L), listener::onFailure)
+            ActionListener.wrap(r -> listener.onResponse(r.getHits().getTotalHits().value() > 0L), listener::onFailure)
         );
     }
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
index ffc4b48f9cc30..9d5175922c892 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
@@ -509,7 +509,7 @@ public void expandTransformIds(
         final ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(idTokens, allowNoMatch);
 
         executeAsyncWithOrigin(request, foundConfigsListener.<SearchResponse>delegateFailureAndWrap((l, searchResponse) -> {
-            long totalHits = searchResponse.getHits().getTotalHits().value;
+            long totalHits = searchResponse.getHits().getTotalHits().value();
             // important: preserve order
             Set<String> ids = Sets.newLinkedHashSetWithExpectedSize(searchResponse.getHits().getHits().length);
             Set<TransformConfig> configs = Sets.newLinkedHashSetWithExpectedSize(searchResponse.getHits().getHits().length);
@@ -589,7 +589,7 @@ public void resetTransform(String transformId, ActionListener<Boolean> listener)
                     .trackTotalHitsUpTo(1)
             );
         executeAsyncWithOrigin(TransportSearchAction.TYPE, searchRequest, deleteListener.delegateFailureAndWrap((l, searchResponse) -> {
-            if (searchResponse.getHits().getTotalHits().value == 0) {
+            if (searchResponse.getHits().getTotalHits().value() == 0) {
                 listener.onFailure(
                     new ResourceNotFoundException(TransformMessages.getMessage(TransformMessages.REST_UNKNOWN_TRANSFORM, transformId))
                 );
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/common/AbstractCompositeAggFunction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/common/AbstractCompositeAggFunction.java
index 23bab56de5ec9..2de810b2b902d 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/common/AbstractCompositeAggFunction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/common/AbstractCompositeAggFunction.java
@@ -207,7 +207,7 @@ private SearchRequest buildSearchRequestForValidation(String logId, SourceConfig
 
     @Override
     public void getInitialProgressFromResponse(SearchResponse response, ActionListener<TransformProgress> progressListener) {
-        progressListener.onResponse(new TransformProgress(response.getHits().getTotalHits().value, 0L, 0L));
+        progressListener.onResponse(new TransformProgress(response.getHits().getTotalHits().value(), 0L, 0L));
     }
 
 }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java
index 684e3a085405d..68b31d4f466b6 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.transform.transforms.pivot;
 
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -560,7 +560,7 @@ static class GeoTileFieldCollector implements FieldCollector {
         @Override
         public int getMaxPageSize() {
             // this collector is limited by indices.query.bool.max_clause_count, default 1024
-            return BooleanQuery.getMaxClauseCount();
+            return IndexSearcher.getMaxClauseCount();
         }
 
         @Override
diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/settings_endpoints/10_watcher_settings.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/settings_endpoints/10_watcher_settings.yml
new file mode 100644
index 0000000000000..f639b4f8f1a77
--- /dev/null
+++ b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/settings_endpoints/10_watcher_settings.yml
@@ -0,0 +1,104 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+  - do:
+      watcher.put_watch:
+        id: "my_watch"
+        body: >
+          {
+            "trigger": {
+              "schedule": {
+                "hourly": {
+                  "minute": [ 0, 5 ]
+                  }
+                }
+            },
+            "input": {
+              "simple": {
+                "payload": {
+                  "send": "yes"
+                }
+              }
+            },
+            "condition": {
+              "always": {}
+            },
+            "actions": {
+              "test_index": {
+                "index": {
+                  "index": "test"
+                }
+              }
+            }
+          }
+
+---
+"Test update and get watch settings api":
+  - do:
+      watcher.get_settings: { }
+
+  - match: { index.auto_expand_replicas: "0-1" }
+  - match: { index.number_of_replicas: "0" }
+
+  - do:
+      watcher.update_settings:
+        body:
+          index.auto_expand_replicas: "0-all"
+
+  - do:
+      watcher.get_settings: { }
+
+  - match: { index.auto_expand_replicas: "0-all" }
+  - is_false: index.routing.allocation.include._tier_preference
+
+  - do:
+      watcher.update_settings:
+        body:
+          index.auto_expand_replicas: null
+          index.number_of_replicas: 1
+
+  - do:
+      watcher.get_settings: { }
+
+  - match: { index.number_of_replicas: "1" }
+---
+"Test disallowed setting name throws error":
+  - requires:
+      test_runner_features: regex
+  - do:
+      watcher.update_settings:
+        body:
+          index.disallowed_setting: "some_invalid_value"
+      catch: bad_request
+  - match:
+      error:
+        type: "action_request_validation_exception"
+        reason: '/illegal settings\: \[index.disallowed_setting\].*/'
+---
+"Test allowed prefix setting name":
+  - do:
+      watcher.update_settings:
+        body:
+          index.routing.allocation.include.role: "watcher"
+          index.routing.allocation.exclude.role: "noWatcher"
+          index.routing.allocation.require.role: "mustWatcher"
+  - do:
+      watcher.get_settings: { }
+  - match: { index.routing.allocation.include.role: "watcher" }
+  - match: { index.routing.allocation.exclude.role: "noWatcher" }
+  - match: { index.routing.allocation.require.role: "mustWatcher" }
+---
+"Test explicitly disallowed prefix setting name throws error":
+  - requires:
+      test_runner_features: regex
+  - do:
+      watcher.update_settings:
+        body:
+          index.routing.allocation.include.disallowed_prefix: "some_invalid_value"
+      catch: bad_request
+  - match:
+      error:
+        type: "action_request_validation_exception"
+        reason: '/illegal settings\: \[index.routing.allocation.include.disallowed_prefix\].*/'
diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/update_settings/10_update_watcher_settings.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/update_settings/10_update_watcher_settings.yml
deleted file mode 100644
index d7478d643a98a..0000000000000
--- a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/update_settings/10_update_watcher_settings.yml
+++ /dev/null
@@ -1,66 +0,0 @@
----
-setup:
-  - do:
-      cluster.health:
-          wait_for_status: yellow
-
----
-"Test update and get watch settings api":
-  - do:
-      watcher.put_watch:
-        id: "my_watch"
-        body:  >
-          {
-            "trigger": {
-              "schedule": {
-                "hourly": {
-                  "minute": [ 0, 5 ]
-                  }
-                }
-            },
-            "input": {
-              "simple": {
-                "payload": {
-                  "send": "yes"
-                }
-              }
-            },
-            "condition": {
-              "always": {}
-            },
-            "actions": {
-              "test_index": {
-                "index": {
-                  "index": "test"
-                }
-              }
-            }
-          }
-  - match: { _id: "my_watch" }
-
-  - do:
-      watcher.get_settings: {}
-
-  -  match: { index.auto_expand_replicas: "0-1" }
-  -  match: { index.number_of_replicas: "0" }
-
-  - do:
-      watcher.update_settings:
-        body:
-          index.auto_expand_replicas: "0-all"
-
-  - do:
-      watcher.get_settings: {}
-
-  -  match: { index.auto_expand_replicas: "0-all" }
-
-  - do:
-      watcher.update_settings:
-        body:
-          index.auto_expand_replicas: null
-          index.number_of_replicas: 1
-
-  - do:
-      watcher.get_settings: {}
-
-  -  match: { index.number_of_replicas: "1" }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java
index 706337768a299..5f7c6490e51f1 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java
@@ -50,7 +50,7 @@ public void testCanUseAnyConcreteIndexName() throws Exception {
         assertBusy(() -> {
             assertResponse(
                 prepareSearch(watchResultsIndex).setTrackTotalHits(true),
-                searchResponse -> assertThat((int) searchResponse.getHits().getTotalHits().value, greaterThan(0))
+                searchResponse -> assertThat((int) searchResponse.getHits().getTotalHits().value(), greaterThan(0))
             );
         });
     }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
index f1ad29607b5b8..7fa5365afa0ab 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
@@ -120,7 +120,7 @@ private void assertTotalHistoryEntries(String id, long expectedCount) throws Exc
             assertResponse(
                 prepareSearch(HistoryStoreField.DATA_STREAM + "*").setSize(0)
                     .setSource(new SearchSourceBuilder().query(QueryBuilders.boolQuery().must(termQuery("watch_id", id)))),
-                searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, is(oneOf(expectedCount, expectedCount + 1)))
+                searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), is(oneOf(expectedCount, expectedCount + 1)))
             );
         });
 
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
index 60867ba5d4410..4068c534013b9 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
@@ -122,7 +122,7 @@ public void testActionConditionWithHardFailures() throws Exception {
             ensureGreen(HistoryStoreField.DATA_STREAM);
             final SearchResponse response = searchHistory(SearchSourceBuilder.searchSource().query(termQuery("watch_id", id)));
             try {
-                assertThat(response.getHits().getTotalHits().value, is(oneOf(1L, 2L)));
+                assertThat(response.getHits().getTotalHits().value(), is(oneOf(1L, 2L)));
                 searchHitReference.set(response.getHits().getAt(0).asUnpooled());
             } finally {
                 response.decRef();
@@ -176,7 +176,7 @@ public void testActionConditionWithFailures() throws Exception {
             ensureGreen(HistoryStoreField.DATA_STREAM);
             final SearchResponse response = searchHistory(SearchSourceBuilder.searchSource().query(termQuery("watch_id", id)));
             try {
-                assertThat(response.getHits().getTotalHits().value, is(oneOf(1L, 2L)));
+                assertThat(response.getHits().getTotalHits().value(), is(oneOf(1L, 2L)));
                 searchHitReference.set(response.getHits().getAt(0).asUnpooled());
             } finally {
                 response.decRef();
@@ -236,7 +236,7 @@ public void testActionCondition() throws Exception {
             ensureGreen(HistoryStoreField.DATA_STREAM);
             final SearchResponse response = searchHistory(SearchSourceBuilder.searchSource().query(termQuery("watch_id", id)));
             try {
-                assertThat(response.getHits().getTotalHits().value, is(oneOf(1L, 2L)));
+                assertThat(response.getHits().getTotalHits().value(), is(oneOf(1L, 2L)));
                 searchHitReference.set(response.getHits().getAt(0).asUnpooled());
             } finally {
                 response.decRef();
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
index 5b7ea39079f28..dac87eaa6f034 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
@@ -105,7 +105,7 @@ public void testEmailFields() throws Exception {
             ),
             response -> {
                 assertThat(response, notNullValue());
-                assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+                assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
                 InternalAggregations aggs = response.getAggregations();
                 assertThat(aggs, notNullValue());
 
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
index 97347de1ea23e..ffac36846414e 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
@@ -102,7 +102,7 @@ public void testHttpFields() throws Exception {
             ),
             response -> {
                 assertThat(response, notNullValue());
-                assertThat(response.getHits().getTotalHits().value, is(oneOf(1L, 2L)));
+                assertThat(response.getHits().getTotalHits().value(), is(oneOf(1L, 2L)));
                 InternalAggregations aggs = response.getAggregations();
                 assertThat(aggs, notNullValue());
 
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
index 7dde279fb90db..8dec5287ae607 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
@@ -54,7 +54,7 @@ public void testIndexActionFields() throws Exception {
             ),
             response -> {
                 assertThat(response, notNullValue());
-                assertThat(response.getHits().getTotalHits().value, is(oneOf(1L, 2L)));
+                assertThat(response.getHits().getTotalHits().value(), is(oneOf(1L, 2L)));
                 InternalAggregations aggs = response.getAggregations();
                 assertThat(aggs, notNullValue());
 
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
index 567d4acfa45e5..b268caa45f471 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
@@ -72,7 +72,7 @@ public void testHttpFields() throws Exception {
             ),
             response -> {
                 assertThat(response, notNullValue());
-                assertThat(response.getHits().getTotalHits().value, is(oneOf(1L, 2L)));
+                assertThat(response.getHits().getTotalHits().value(), is(oneOf(1L, 2L)));
                 InternalAggregations aggs = response.getAggregations();
                 assertThat(aggs, notNullValue());
 
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
index 5dc537fc259d9..5eaf27e7b2670 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
@@ -377,7 +377,7 @@ protected void assertWatchWithMinimumPerformedActionsCount(
                         lastResponse.set(searchResponse);
                         assertThat(
                             "could not find executed watch record for watch " + watchName,
-                            searchResponse.getHits().getTotalHits().value,
+                            searchResponse.getHits().getTotalHits().value(),
                             greaterThanOrEqualTo(minimumExpectedWatchActionsWithActionPerformed)
                         );
                         if (assertConditionMet) {
@@ -396,7 +396,7 @@ protected void assertWatchWithMinimumPerformedActionsCount(
         } catch (AssertionError error) {
             SearchResponse searchResponse = lastResponse.get();
             try {
-                logger.info("Found [{}] records for watch [{}]", searchResponse.getHits().getTotalHits().value, watchName);
+                logger.info("Found [{}] records for watch [{}]", searchResponse.getHits().getTotalHits().value(), watchName);
                 int counter = 1;
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
                     logger.info("hit [{}]=\n {}", counter++, XContentHelper.convertToJson(hit.getSourceRef(), true, true));
@@ -452,7 +452,7 @@ protected void assertWatchWithNoActionNeeded(final String watchName, final long
                     searchResponse -> {
                         lastResponse.set(searchResponse);
                         assertThat(
-                            searchResponse.getHits().getTotalHits().value,
+                            searchResponse.getHits().getTotalHits().value(),
                             greaterThanOrEqualTo(expectedWatchActionsWithNoActionNeeded)
                         );
                     }
@@ -461,7 +461,7 @@ protected void assertWatchWithNoActionNeeded(final String watchName, final long
         } catch (AssertionError error) {
             SearchResponse searchResponse = lastResponse.get();
             try {
-                logger.info("Found [{}] records for watch [{}]", searchResponse.getHits().getTotalHits().value, watchName);
+                logger.info("Found [{}] records for watch [{}]", searchResponse.getHits().getTotalHits().value(), watchName);
                 int counter = 1;
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
                     logger.info("hit [{}]=\n {}", counter++, XContentHelper.convertToJson(hit.getSourceRef(), true, true));
@@ -497,7 +497,7 @@ protected void assertWatchWithMinimumActionsCount(final String watchName, final
                 searchResponse -> {
                     assertThat(
                         "could not find executed watch record",
-                        searchResponse.getHits().getTotalHits().value,
+                        searchResponse.getHits().getTotalHits().value(),
                         greaterThanOrEqualTo(recordCount)
                     );
                 }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java
index 99640d1ebc3ea..03f1e6cb57eb8 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java
@@ -296,8 +296,8 @@ private void assertSingleExecutionAndCompleteWatchHistory(final long numberOfWat
             AtomicLong successfulWatchExecutions = new AtomicLong();
             refresh();
             assertResponse(prepareSearch("output"), searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, is(greaterThanOrEqualTo(numberOfWatches)));
-                successfulWatchExecutions.set(searchResponse.getHits().getTotalHits().value);
+                assertThat(searchResponse.getHits().getTotalHits().value(), is(greaterThanOrEqualTo(numberOfWatches)));
+                successfulWatchExecutions.set(searchResponse.getHits().getTotalHits().value());
             });
 
             // the watch history should contain entries for each triggered watch, which a few have been marked as not executed
@@ -378,7 +378,7 @@ public void testWatchRecordSavedTwice() throws Exception {
             // the actual documents are in the output index
             refresh();
             assertResponse(prepareSearch(HistoryStoreField.DATA_STREAM).setSize(numRecords), searchResponse -> {
-                assertThat(searchResponse.getHits().getTotalHits().value, Matchers.equalTo((long) numRecords));
+                assertThat(searchResponse.getHits().getTotalHits().value(), Matchers.equalTo((long) numRecords));
                 for (int i = 0; i < numRecords; i++) {
                     assertThat(searchResponse.getHits().getAt(i).getSourceAsMap().get("state"), is(ExecutionState.EXECUTED.id()));
                 }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java
index e5f4091ca89eb..f3648580691cb 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java
@@ -55,7 +55,7 @@ public void testHistoryOnRejection() throws Exception {
         assertBusy(() -> {
             flushAndRefresh(".watcher-history-*");
             assertResponse(prepareSearch(".watcher-history-*"), searchResponse -> {
-                assertThat("Watcher history not found", searchResponse.getHits().getTotalHits().value, greaterThanOrEqualTo(2L));
+                assertThat("Watcher history not found", searchResponse.getHits().getTotalHits().value(), greaterThanOrEqualTo(2L));
                 assertThat(
                     "Did not find watcher history for rejected watch",
                     Arrays.stream(searchResponse.getHits().getHits())
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
index 7ff293ed9b150..fbb1996a4cf42 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
@@ -69,7 +69,7 @@ public void testThatLoadingWithNonExistingIndexWorks() throws Exception {
             assertThat(refreshResponse.getStatus(), equalTo(RestStatus.OK));
             assertResponse(
                 prepareSearch(".watcher-history*").setSize(0),
-                searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value, is(greaterThanOrEqualTo(1L)))
+                searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), is(greaterThanOrEqualTo(1L)))
             );
         }, 30, TimeUnit.SECONDS);
     }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
index 4298f641cbdd2..e12805f3ace09 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
@@ -79,7 +79,7 @@ public void testWatchMetadata() throws Exception {
             }
             assertNotNull(searchResponse);
             try {
-                assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
+                assertThat(searchResponse.getHits().getTotalHits().value(), greaterThan(0L));
             } finally {
                 searchResponse.decRef();
             }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
index 92ac91a63e097..2ec6541275d04 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
@@ -139,13 +139,13 @@ public void testScriptTransform() throws Exception {
         refresh();
 
         assertNoFailuresAndResponse(prepareSearch("output1"), response -> {
-            assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
             assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getSourceAsMap().get("key3").toString(), equalTo("20"));
         });
 
         assertNoFailuresAndResponse(prepareSearch("output2"), response -> {
-            assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
             assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getSourceAsMap().get("key3").toString(), equalTo("20"));
         });
@@ -184,12 +184,12 @@ public void testSearchTransform() throws Exception {
         refresh();
 
         assertNoFailuresAndResponse(prepareSearch("output1"), response -> {
-            assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
             assertThat(response.getHits().getAt(0).getSourceAsString(), containsString("mytestresult"));
         });
 
         assertNoFailuresAndResponse(prepareSearch("output2"), response -> {
-            assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
             assertThat(response.getHits().getAt(0).getSourceAsString(), containsString("mytestresult"));
         });
     }
@@ -223,13 +223,13 @@ public void testChainTransform() throws Exception {
         refresh();
 
         assertNoFailuresAndResponse(prepareSearch("output1"), response -> {
-            assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
             assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getSourceAsMap().get("key4").toString(), equalTo("30"));
         });
 
         assertNoFailuresAndResponse(prepareSearch("output2"), response -> {
-            assertThat(response.getHits().getTotalHits().value, greaterThanOrEqualTo(1L));
+            assertThat(response.getHits().getTotalHits().value(), greaterThanOrEqualTo(1L));
             assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1));
             assertThat(response.getHits().getAt(0).getSourceAsMap().get("key4").toString(), equalTo("30"));
         });
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
index 5389f34212270..0ea9b432d3b0f 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
@@ -340,7 +340,7 @@ private Collection<Watch> loadWatches(ClusterState clusterState) {
                 throw new ElasticsearchException("Partial response while loading watches");
             }
 
-            if (response.getHits().getTotalHits().value == 0) {
+            if (response.getHits().getTotalHits().value() == 0) {
                 return Collections.emptyList();
             }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
index 9a165112c41d1..327d345af864e 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
@@ -42,7 +42,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -440,7 +439,7 @@ static CharacterRunAutomaton createAutomaton(List<String> whiteListedHosts) {
         }
 
         Automaton whiteListAutomaton = Regex.simpleMatchToAutomaton(whiteListedHosts.toArray(Strings.EMPTY_ARRAY));
-        whiteListAutomaton = MinimizationOperations.minimize(whiteListAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        whiteListAutomaton = Operations.determinize(whiteListAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
         return new CharacterRunAutomaton(whiteListAutomaton);
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java
index 6775dca424bf1..dfa0c47493ed7 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java
@@ -156,7 +156,7 @@ public Collection<TriggeredWatch> findTriggeredWatches(Collection<Watch> watches
         SearchResponse response = null;
         try {
             response = client.search(searchRequest).actionGet(defaultSearchTimeout);
-            logger.debug("trying to find triggered watches for ids {}: found [{}] docs", ids, response.getHits().getTotalHits().value);
+            logger.debug("trying to find triggered watches for ids {}: found [{}] docs", ids, response.getHits().getTotalHits().value());
             while (response.getHits().getHits().length != 0) {
                 for (SearchHit hit : response.getHits()) {
                     Wid wid = new Wid(hit.getId());
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java
index 9d6186e9c1c48..e6bd1b0efb95d 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java
@@ -91,7 +91,7 @@ SearchInput.Result doExecute(WatchExecutionContext ctx, WatcherSearchTemplateReq
         try {
 
             if (logger.isDebugEnabled()) {
-                logger.debug("[{}] found [{}] hits", ctx.id(), response.getHits().getTotalHits().value);
+                logger.debug("[{}] found [{}] hits", ctx.id(), response.getHits().getTotalHits().value());
             }
 
             final Payload payload;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java
index f1a6d7b07d8e7..79470f967ab3c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java
@@ -141,7 +141,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (priority != null) {
             builder.field(Field.PRIORITY.getPreferredName(), priority.value());
         }
-        builder.timeField(Field.SENT_DATE.getPreferredName(), sentDate);
+        builder.timestampField(Field.SENT_DATE.getPreferredName(), sentDate);
         if (to != null) {
             builder.field(Field.TO.getPreferredName(), to, params);
         }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java
index a01bf660ba8d5..f298a1cf00624 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -34,14 +33,10 @@ public class RestAckWatchAction extends BaseRestHandler {
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(POST, "/_watcher/watch/{id}/_ack").replaces(POST, "/_xpack/watcher/watch/{id}/_ack", RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_watcher/watch/{id}/_ack").replaces(PUT, "/_xpack/watcher/watch/{id}/_ack", RestApiVersion.V_7).build(),
-            Route.builder(POST, "/_watcher/watch/{id}/_ack/{actions}")
-                .replaces(POST, "/_xpack/watcher/watch/{id}/_ack/{actions}", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_watcher/watch/{id}/_ack/{actions}")
-                .replaces(PUT, "/_xpack/watcher/watch/{id}/_ack/{actions}", RestApiVersion.V_7)
-                .build()
+            new Route(POST, "/_watcher/watch/{id}/_ack"),
+            new Route(PUT, "/_watcher/watch/{id}/_ack"),
+            new Route(POST, "/_watcher/watch/{id}/_ack/{actions}"),
+            new Route(PUT, "/_watcher/watch/{id}/_ack/{actions}")
         );
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java
index 18c9e5a2825f4..b8b466883f7fe 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -33,14 +32,7 @@ public class RestActivateWatchAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_watcher/watch/{id}/_activate")
-                .replaces(POST, "/_xpack/watcher/watch/{id}/_activate", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_watcher/watch/{id}/_activate")
-                .replaces(PUT, "/_xpack/watcher/watch/{id}/_activate", RestApiVersion.V_7)
-                .build()
-        );
+        return List.of(new Route(POST, "/_watcher/watch/{id}/_activate"), new Route(PUT, "/_watcher/watch/{id}/_activate"));
     }
 
     @Override
@@ -72,14 +64,7 @@ public static class DeactivateRestHandler extends BaseRestHandler {
 
         @Override
         public List<Route> routes() {
-            return List.of(
-                Route.builder(POST, "/_watcher/watch/{id}/_deactivate")
-                    .replaces(POST, "/_xpack/watcher/watch/{id}/_deactivate", RestApiVersion.V_7)
-                    .build(),
-                Route.builder(PUT, "/_watcher/watch/{id}/_deactivate")
-                    .replaces(PUT, "/_xpack/watcher/watch/{id}/_deactivate", RestApiVersion.V_7)
-                    .build()
-            );
+            return List.of(new Route(POST, "/_watcher/watch/{id}/_deactivate"), new Route(PUT, "/_watcher/watch/{id}/_deactivate"));
         }
 
         @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
index 3c2ddc7a254d1..e909a34157583 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -29,9 +28,7 @@ public class RestDeleteWatchAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(DELETE, "/_watcher/watch/{id}").replaces(DELETE, "/_xpack/watcher/watch/{id}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(DELETE, "/_watcher/watch/{id}"));
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java
index 56d249c427be1..a25e7704c0653 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequestFilter;
@@ -55,14 +54,10 @@ public class RestExecuteWatchAction extends BaseRestHandler implements RestReque
     @Override
     public List<Route> routes() {
         return List.of(
-            Route.builder(POST, "/_watcher/watch/{id}/_execute")
-                .replaces(POST, "/_xpack/watcher/watch/{id}/_execute", RestApiVersion.V_7)
-                .build(),
-            Route.builder(PUT, "/_watcher/watch/{id}/_execute")
-                .replaces(PUT, "/_xpack/watcher/watch/{id}/_execute", RestApiVersion.V_7)
-                .build(),
-            Route.builder(POST, "/_watcher/watch/_execute").replaces(POST, "/_xpack/watcher/watch/_execute", RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_watcher/watch/_execute").replaces(PUT, "/_xpack/watcher/watch/_execute", RestApiVersion.V_7).build()
+            new Route(POST, "/_watcher/watch/{id}/_execute"),
+            new Route(PUT, "/_watcher/watch/{id}/_execute"),
+            new Route(POST, "/_watcher/watch/_execute"),
+            new Route(PUT, "/_watcher/watch/_execute")
         );
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java
index 5467297ab1e72..f296e9869cd9e 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -29,7 +28,7 @@ public class RestGetWatchAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(GET, "/_watcher/watch/{id}").replaces(GET, "/_xpack/watcher/watch/{id}", RestApiVersion.V_7).build());
+        return List.of(new Route(GET, "/_watcher/watch/{id}"));
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java
index 73a933c9c2e46..ff9e8c45d72f3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java
@@ -17,6 +17,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
 /**
  * Allows retrieving a subset of index settings (those use-settable) for the .watches index.
  * See {@link RestUpdateWatcherSettingsAction} for the setting counterpart.
@@ -29,7 +31,7 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(RestRequest.Method.GET, "/_watcher/settings").build());
+        return List.of(new Route(GET, "/_watcher/settings"));
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
index 42820dabf9d7d..9dba72b1f64c3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -33,10 +32,7 @@ public class RestPutWatchAction extends BaseRestHandler implements RestRequestFi
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(POST, "/_watcher/watch/{id}").replaces(POST, "/_xpack/watcher/watch/{id}", RestApiVersion.V_7).build(),
-            Route.builder(PUT, "/_watcher/watch/{id}").replaces(PUT, "/_xpack/watcher/watch/{id}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(POST, "/_watcher/watch/{id}"), new Route(PUT, "/_watcher/watch/{id}"));
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java
index 26f64a0918141..db14a6572fa99 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java
@@ -17,6 +17,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
 /**
  * Allows setting a subset of index settings for the .watches index.
  * See {@link RestGetWatcherSettingsAction} for the retrieval counterpart.
@@ -29,7 +31,7 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(RestRequest.Method.PUT, "/_watcher/settings").build());
+        return List.of(new Route(PUT, "/_watcher/settings"));
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
index f6b9ae9c4031c..a52e0882fa81e 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestUtils;
@@ -24,7 +23,7 @@ public class RestWatchServiceAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(Route.builder(POST, "/_watcher/_start").replaces(POST, "/_xpack/watcher/_start", RestApiVersion.V_7).build());
+        return List.of(new Route(POST, "/_watcher/_start"));
     }
 
     @Override
@@ -42,7 +41,7 @@ public static class StopRestHandler extends BaseRestHandler {
 
         @Override
         public List<Route> routes() {
-            return List.of(Route.builder(POST, "/_watcher/_stop").replaces(POST, "/_xpack/watcher/_stop", RestApiVersion.V_7).build());
+            return List.of(new Route(POST, "/_watcher/_stop"));
         }
 
         @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
index f7a59553a65ff..2c04501ee66d6 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
@@ -29,10 +28,7 @@ public class RestWatcherStatsAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(
-            Route.builder(GET, "/_watcher/stats").replaces(GET, "/_xpack/watcher/stats", RestApiVersion.V_7).build(),
-            Route.builder(GET, "/_watcher/stats/{metric}").replaces(GET, "/_xpack/watcher/stats/{metric}", RestApiVersion.V_7).build()
-        );
+        return List.of(new Route(GET, "/_watcher/stats"), new Route(GET, "/_watcher/stats/{metric}"));
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
index 29349735afcd2..2962bffd68b66 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
@@ -23,8 +23,10 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.GetWatcherSettingsAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction;
 
+import static org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction.ALLOWED_SETTINGS_PREFIXES;
+import static org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction.ALLOWED_SETTING_KEYS;
+import static org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction.EXPLICITLY_DENIED_SETTINGS;
 import static org.elasticsearch.xpack.watcher.transport.actions.TransportUpdateWatcherSettingsAction.WATCHER_INDEX_NAME;
 import static org.elasticsearch.xpack.watcher.transport.actions.TransportUpdateWatcherSettingsAction.WATCHER_INDEX_REQUEST;
 
@@ -73,11 +75,14 @@ protected void masterOperation(
      */
     private static Settings filterSettableSettings(Settings settings) {
         Settings.Builder builder = Settings.builder();
-        for (String settingName : UpdateWatcherSettingsAction.ALLOWED_SETTING_KEYS) {
-            if (settings.hasValue(settingName)) {
-                builder.put(settingName, settings.get(settingName));
-            }
-        }
+        settings.keySet()
+            .stream()
+            .filter(
+                setting -> (ALLOWED_SETTING_KEYS.contains(setting)
+                    || ALLOWED_SETTINGS_PREFIXES.stream().anyMatch(prefix -> setting.startsWith(prefix + ".")))
+                    && EXPLICITLY_DENIED_SETTINGS.contains(setting) == false
+            )
+            .forEach(setting -> builder.put(setting, settings.get(setting)));
         return builder.build();
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java
index 97ae29a26e68c..358a839e60ea5 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java
@@ -98,11 +98,11 @@ SearchRequest createSearchRequest(QueryWatchesAction.Request request) {
     }
 
     void transformResponse(SearchResponse searchResponse, ActionListener<QueryWatchesAction.Response> listener) {
-        assert searchResponse.getHits().getTotalHits().relation == TotalHits.Relation.EQUAL_TO;
+        assert searchResponse.getHits().getTotalHits().relation() == TotalHits.Relation.EQUAL_TO;
         List<QueryWatchesAction.Response.Item> items = Arrays.stream(searchResponse.getHits().getHits())
             .map(this::transformSearchHit)
             .toList();
-        listener.onResponse(new QueryWatchesAction.Response(searchResponse.getHits().getTotalHits().value, items));
+        listener.onResponse(new QueryWatchesAction.Response(searchResponse.getHits().getTotalHits().value(), items));
     }
 
     QueryWatchesAction.Response.Item transformSearchHit(SearchHit searchHit) {
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java
index ced131640f0ee..cc8d0edf37014 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.xpack.core.watcher.support.WatcherDateTimeUtils;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
+import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
 import org.elasticsearch.xpack.watcher.trigger.schedule.Schedule;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleRegistry;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger;
@@ -32,6 +34,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -67,7 +70,11 @@ public synchronized void start(Collection<Watch> jobs) {
         Map<String, ActiveSchedule> startingSchedules = Maps.newMapWithExpectedSize(jobs.size());
         for (Watch job : jobs) {
             if (job.trigger() instanceof ScheduleTrigger trigger) {
-                startingSchedules.put(job.id(), new ActiveSchedule(job.id(), trigger.getSchedule(), startTime));
+                if (trigger.getSchedule() instanceof IntervalSchedule) {
+                    startingSchedules.put(job.id(), new ActiveSchedule(job.id(), trigger.getSchedule(), calculateLastStartTime(job)));
+                } else {
+                    startingSchedules.put(job.id(), new ActiveSchedule(job.id(), trigger.getSchedule(), startTime));
+                }
             }
         }
         // why are we calling putAll() here instead of assigning a brand
@@ -108,10 +115,39 @@ public void add(Watch watch) {
         // watcher indexing listener
         // this also means that updating an existing watch would not retrigger the schedule time, if it remains the same schedule
         if (currentSchedule == null || currentSchedule.schedule.equals(trigger.getSchedule()) == false) {
-            schedules.put(watch.id(), new ActiveSchedule(watch.id(), trigger.getSchedule(), clock.millis()));
+            if (trigger.getSchedule() instanceof IntervalSchedule) {
+                schedules.put(watch.id(), new ActiveSchedule(watch.id(), trigger.getSchedule(), calculateLastStartTime(watch)));
+            } else {
+                schedules.put(watch.id(), new ActiveSchedule(watch.id(), trigger.getSchedule(), clock.millis()));
+            }
+
         }
     }
 
+    /**
+     * Attempts to calculate the epoch millis of the last time the watch was checked, If the watch has never been checked, the timestamp of
+     * the last state change is used. If the watch has never been checked and has never been in an active state, the current time is used.
+     * @param job the watch to calculate the last start time for
+     * @return the epoch millis of the last time the watch was checked or now
+     */
+    private long calculateLastStartTime(Watch job) {
+        var lastChecked = Optional.ofNullable(job)
+            .map(Watch::status)
+            .map(WatchStatus::lastChecked)
+            .map(ZonedDateTime::toInstant)
+            .map(Instant::toEpochMilli);
+
+        return lastChecked.orElseGet(
+            () -> Optional.ofNullable(job)
+                .map(Watch::status)
+                .map(WatchStatus::state)
+                .map(WatchStatus.State::getTimestamp)
+                .map(ZonedDateTime::toInstant)
+                .map(Instant::toEpochMilli)
+                .orElse(clock.millis())
+        );
+    }
+
     @Override
     public boolean remove(String jobId) {
         logger.debug("Removing watch [{}] from engine (engine is running: {})", jobId, isRunning.get());
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java
index 8b7cfa75f9229..ef290628c06d5 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.input.none.ExecutableNoneInput;
 import org.elasticsearch.xpack.watcher.trigger.schedule.Schedule;
@@ -283,6 +284,240 @@ public void testAddOnlyWithNewSchedule() {
         assertThat(engine.getSchedules().get("_id"), not(is(activeSchedule)));
     }
 
+    /**
+     * This test verifies that a watch with a valid lastCheckedTime executes before the interval time to ensure the job resumes waiting
+     * from the same point it left off before the reallocation / restart
+     */
+    public void testWatchWithLastCheckedTimeExecutesBeforeInitialInterval() throws Exception {
+        final var firstLatch = new CountDownLatch(1);
+        final var secondLatch = new CountDownLatch(1);
+
+        Watch watch = new Watch(
+            "watch",
+            new ScheduleTrigger(interval("1s")),
+            new ExecutableNoneInput(),
+            InternalAlwaysCondition.INSTANCE,
+            null,
+            null,
+            Collections.emptyList(),
+            null,
+            new WatchStatus(-1L, null, null, clock.instant().minusMillis(500).atZone(ZoneOffset.UTC), null, null, null),
+            SequenceNumbers.UNASSIGNED_SEQ_NO,
+            SequenceNumbers.UNASSIGNED_PRIMARY_TERM
+        );
+
+        var watches = Collections.singletonList(watch);
+
+        var runCount = new AtomicInteger(0);
+
+        engine.register(events -> {
+            for (TriggerEvent ignored : events) {
+                if (runCount.getAndIncrement() == 0) {
+                    logger.info("job first fire");
+                    firstLatch.countDown();
+                } else {
+                    logger.info("job second fire");
+                    secondLatch.countDown();
+                }
+            }
+        });
+
+        engine.start(watches);
+        advanceClockIfNeeded(clock.instant().plusMillis(510).atZone(ZoneOffset.UTC));
+        if (firstLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        advanceClockIfNeeded(clock.instant().plusMillis(1100).atZone(ZoneOffset.UTC));
+        if (secondLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        assertThat(runCount.get(), is(2));
+
+        engine.stop();
+    }
+
+    /**
+     * This test verifies that a watch without a lastCheckedTime but with a valid activationTime executes before the interval time to
+     * ensure the job resumes waiting from the same point it left off before the reallocation / restart
+     */
+    public void testWatchWithNoLastCheckedTimeButHasActivationTimeExecutesBeforeInitialInterval() throws Exception {
+        final var firstLatch = new CountDownLatch(1);
+        final var secondLatch = new CountDownLatch(1);
+
+        Watch watch = new Watch(
+            "watch",
+            new ScheduleTrigger(interval("1s")),
+            new ExecutableNoneInput(),
+            InternalAlwaysCondition.INSTANCE,
+            null,
+            null,
+            Collections.emptyList(),
+            null,
+            new WatchStatus(
+                -1L,
+                new WatchStatus.State(true, clock.instant().minusMillis(500).atZone(ZoneOffset.UTC)),
+                null,
+                null,
+                null,
+                null,
+                null
+            ),
+            SequenceNumbers.UNASSIGNED_SEQ_NO,
+            SequenceNumbers.UNASSIGNED_PRIMARY_TERM
+        );
+
+        var watches = Collections.singletonList(watch);
+
+        var runCount = new AtomicInteger(0);
+
+        engine.register(events -> {
+            for (TriggerEvent ignored : events) {
+                if (runCount.getAndIncrement() == 0) {
+                    logger.info("job first fire");
+                    firstLatch.countDown();
+                } else {
+                    logger.info("job second fire");
+                    secondLatch.countDown();
+                }
+            }
+        });
+
+        engine.start(watches);
+        advanceClockIfNeeded(clock.instant().plusMillis(510).atZone(ZoneOffset.UTC));
+        if (firstLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        advanceClockIfNeeded(clock.instant().plusMillis(1100).atZone(ZoneOffset.UTC));
+        if (secondLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        assertThat(runCount.get(), is(2));
+
+        engine.stop();
+    }
+
+    /**
+     * This test verifies that a watch added after service start with a lastCheckedTime executes before the interval time to ensure the job
+     * resumes waiting from the same point it left off before the reallocation / restart
+     */
+    public void testAddWithLastCheckedTimeExecutesBeforeInitialInterval() throws Exception {
+        final var firstLatch = new CountDownLatch(1);
+        final var secondLatch = new CountDownLatch(1);
+
+        Watch watch = new Watch(
+            "watch",
+            new ScheduleTrigger(interval("1s")),
+            new ExecutableNoneInput(),
+            InternalAlwaysCondition.INSTANCE,
+            null,
+            null,
+            Collections.emptyList(),
+            null,
+            new WatchStatus(-1L, null, null, clock.instant().minusMillis(500).atZone(ZoneOffset.UTC), null, null, null),
+            SequenceNumbers.UNASSIGNED_SEQ_NO,
+            SequenceNumbers.UNASSIGNED_PRIMARY_TERM
+        );
+
+        var runCount = new AtomicInteger(0);
+
+        engine.register(events -> {
+            for (TriggerEvent ignored : events) {
+                if (runCount.getAndIncrement() == 0) {
+                    logger.info("job first fire");
+                    firstLatch.countDown();
+                } else {
+                    logger.info("job second fire");
+                    secondLatch.countDown();
+                }
+            }
+        });
+
+        engine.start(Collections.emptyList());
+        advanceClockIfNeeded(clock.instant().plusMillis(1100).atZone(ZoneOffset.UTC));
+        engine.add(watch);
+
+        advanceClockIfNeeded(clock.instant().plusMillis(510).atZone(ZoneOffset.UTC));
+        if (firstLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        advanceClockIfNeeded(clock.instant().plusMillis(1100).atZone(ZoneOffset.UTC));
+        if (secondLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        assertThat(runCount.get(), is(2));
+
+        engine.stop();
+    }
+
+    /**
+     * This test verifies that a watch added after service start without a lastCheckedTime but with a valid activationTime executes before
+     * the interval time to ensure the job resumes waiting from the same point it left off before the reallocation / restart
+     */
+    public void testAddWithNoLastCheckedTimeButHasActivationTimeExecutesBeforeInitialInterval() throws Exception {
+        final var firstLatch = new CountDownLatch(1);
+        final var secondLatch = new CountDownLatch(1);
+
+        Watch watch = new Watch(
+            "watch",
+            new ScheduleTrigger(interval("1s")),
+            new ExecutableNoneInput(),
+            InternalAlwaysCondition.INSTANCE,
+            null,
+            null,
+            Collections.emptyList(),
+            null,
+            new WatchStatus(
+                -1L,
+                new WatchStatus.State(true, clock.instant().minusMillis(500).atZone(ZoneOffset.UTC)),
+                null,
+                null,
+                null,
+                null,
+                null
+            ),
+            SequenceNumbers.UNASSIGNED_SEQ_NO,
+            SequenceNumbers.UNASSIGNED_PRIMARY_TERM
+        );
+
+        var runCount = new AtomicInteger(0);
+
+        engine.register(events -> {
+            for (TriggerEvent ignored : events) {
+                if (runCount.getAndIncrement() == 0) {
+                    logger.info("job first fire");
+                    firstLatch.countDown();
+                } else {
+                    logger.info("job second fire");
+                    secondLatch.countDown();
+                }
+            }
+        });
+
+        engine.start(Collections.emptyList());
+        advanceClockIfNeeded(clock.instant().plusMillis(1100).atZone(ZoneOffset.UTC));
+        engine.add(watch);
+
+        advanceClockIfNeeded(clock.instant().plusMillis(510).atZone(ZoneOffset.UTC));
+        if (firstLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        advanceClockIfNeeded(clock.instant().plusMillis(1100).atZone(ZoneOffset.UTC));
+        if (secondLatch.await(3, TimeUnit.SECONDS) == false) {
+            fail("waiting too long for all watches to be triggered");
+        }
+
+        assertThat(runCount.get(), is(2));
+
+        engine.stop();
+    }
+
     private Watch createWatch(String name, Schedule schedule) {
         return new Watch(
             name,
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/BinaryDvConfirmedAutomatonQuery.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/BinaryDvConfirmedAutomatonQuery.java
index 608e5f1972373..191775f46cd72 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/BinaryDvConfirmedAutomatonQuery.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/BinaryDvConfirmedAutomatonQuery.java
@@ -18,6 +18,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BytesRef;
@@ -69,44 +70,56 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
         return new ConstantScoreWeight(this, boost) {
 
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 ByteArrayStreamInput bytes = new ByteArrayStreamInput();
                 final BinaryDocValues values = DocValues.getBinary(context.reader(), field);
-                Scorer approxScorer = approxWeight.scorer(context);
-                if (approxScorer == null) {
+                ScorerSupplier approxScorerSupplier = approxWeight.scorerSupplier(context);
+                if (approxScorerSupplier == null) {
                     // No matches to be had
                     return null;
                 }
-                DocIdSetIterator approxDisi = approxScorer.iterator();
-                TwoPhaseIterator twoPhase = new TwoPhaseIterator(approxDisi) {
+
+                return new ScorerSupplier() {
                     @Override
-                    public boolean matches() throws IOException {
-                        if (values.advanceExact(approxDisi.docID()) == false) {
-                            // Can happen when approxQuery resolves to some form of MatchAllDocs expression
-                            return false;
-                        }
-                        BytesRef arrayOfValues = values.binaryValue();
-                        bytes.reset(arrayOfValues.bytes);
-                        bytes.setPosition(arrayOfValues.offset);
-
-                        int size = bytes.readVInt();
-                        for (int i = 0; i < size; i++) {
-                            int valLength = bytes.readVInt();
-                            if (bytesMatcher.run(arrayOfValues.bytes, bytes.getPosition(), valLength)) {
-                                return true;
+                    public Scorer get(long leadCost) throws IOException {
+                        Scorer approxScorer = approxScorerSupplier.get(leadCost);
+                        DocIdSetIterator approxDisi = approxScorer.iterator();
+                        TwoPhaseIterator twoPhase = new TwoPhaseIterator(approxDisi) {
+                            @Override
+                            public boolean matches() throws IOException {
+                                if (values.advanceExact(approxDisi.docID()) == false) {
+                                    // Can happen when approxQuery resolves to some form of MatchAllDocs expression
+                                    return false;
+                                }
+                                BytesRef arrayOfValues = values.binaryValue();
+                                bytes.reset(arrayOfValues.bytes);
+                                bytes.setPosition(arrayOfValues.offset);
+
+                                int size = bytes.readVInt();
+                                for (int i = 0; i < size; i++) {
+                                    int valLength = bytes.readVInt();
+                                    if (bytesMatcher.run(arrayOfValues.bytes, bytes.getPosition(), valLength)) {
+                                        return true;
+                                    }
+                                    bytes.skipBytes(valLength);
+                                }
+                                return false;
+                            }
+
+                            @Override
+                            public float matchCost() {
+                                // TODO: how can we compute this?
+                                return 1000f;
                             }
-                            bytes.skipBytes(valLength);
-                        }
-                        return false;
+                        };
+                        return new ConstantScoreScorer(score(), scoreMode, twoPhase);
                     }
 
                     @Override
-                    public float matchCost() {
-                        // TODO: how can we compute this?
-                        return 1000f;
+                    public long cost() {
+                        return approxScorerSupplier.cost();
                     }
                 };
-                return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
             }
 
             @Override
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
index 7784e7ffdda12..f3b01bb898126 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -38,7 +38,6 @@
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ElasticsearchParseException;
@@ -349,7 +348,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, boolean
             }
             Automaton automaton = caseInsensitive
                 ? AutomatonQueries.toCaseInsensitiveWildcardAutomaton(new Term(name(), wildcardPattern))
-                : WildcardQuery.toAutomaton(new Term(name(), wildcardPattern));
+                : WildcardQuery.toAutomaton(new Term(name(), wildcardPattern), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
             if (clauseCount > 0) {
                 // We can accelerate execution with the ngram query
                 BooleanQuery approxQuery = rewritten.build();
@@ -379,7 +378,6 @@ public Query regexpQuery(
             RegExp regExp = new RegExp(value, syntaxFlags, matchFlags);
             Automaton a = regExp.toAutomaton();
             a = Operations.determinize(a, maxDeterminizedStates);
-            a = MinimizationOperations.minimize(a, maxDeterminizedStates);
             if (Operations.isTotal(a)) { // Will match all
                 return existsQuery(context);
             }
@@ -390,7 +388,7 @@ public Query regexpQuery(
             Query approxNgramQuery = rewriteBoolToNgramQuery(approxBooleanQuery);
 
             RegExp regex = new RegExp(value, syntaxFlags, matchFlags);
-            Automaton automaton = regex.toAutomaton(maxDeterminizedStates);
+            Automaton automaton = Operations.determinize(regex.toAutomaton(), maxDeterminizedStates);
 
             // We can accelerate execution with the ngram query
             return new BinaryDvConfirmedAutomatonQuery(approxNgramQuery, name(), value, automaton);
@@ -550,9 +548,9 @@ private Query rewriteBoolToNgramQuery(Query approxQuery) {
                 BooleanQuery.Builder rewritten = new BooleanQuery.Builder();
                 int clauseCount = 0;
                 for (BooleanClause clause : bq) {
-                    Query q = rewriteBoolToNgramQuery(clause.getQuery());
+                    Query q = rewriteBoolToNgramQuery(clause.query());
                     if (q != null) {
-                        if (clause.getOccur().equals(Occur.FILTER)) {
+                        if (clause.occur().equals(Occur.FILTER)) {
                             // Can't drop "should" clauses because it can elevate a sibling optional item
                             // to mandatory (shoulds with 1 clause) causing false negatives
                             // Dropping MUSTs increase false positives which are OK because are verified anyway.
@@ -561,7 +559,7 @@ private Query rewriteBoolToNgramQuery(Query approxQuery) {
                                 break;
                             }
                         }
-                        rewritten.add(q, clause.getOccur());
+                        rewritten.add(q, clause.occur());
                     }
                 }
                 return rewritten.build();
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
index 4b9ccff6f526c..a1a01ebdcc590 100644
--- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
+++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
@@ -41,6 +41,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
@@ -182,7 +183,7 @@ public void testTooBigKeywordField() throws IOException {
 
         Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery("*a*", null, null);
         TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
-        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(1L));
+        assertThat(wildcardFieldTopDocs.totalHits.value(), equalTo(1L));
 
         reader.close();
         dir.close();
@@ -229,12 +230,12 @@ public void testTooBigQueryField() throws IOException {
         String queryString = randomABString((IndexSearcher.getMaxClauseCount() * 2) + 1);
         Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(queryString, null, null);
         TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
-        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
+        assertThat(wildcardFieldTopDocs.totalHits.value(), equalTo(0L));
 
         // Test regexp query
         wildcardFieldQuery = wildcardFieldType.fieldType().regexpQuery(queryString, RegExp.ALL, 0, 20000, null, MOCK_CONTEXT);
         wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
-        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
+        assertThat(wildcardFieldTopDocs.totalHits.value(), equalTo(0L));
 
         reader.close();
         dir.close();
@@ -271,13 +272,13 @@ public void testTermAndPrefixQueryIgnoreWildcardSyntax() throws IOException {
     private void expectTermMatch(IndexSearcher searcher, String term, long count) throws IOException {
         Query q = wildcardFieldType.fieldType().termQuery(term, MOCK_CONTEXT);
         TopDocs td = searcher.search(q, 10, Sort.RELEVANCE);
-        assertThat(td.totalHits.value, equalTo(count));
+        assertThat(td.totalHits.value(), equalTo(count));
     }
 
     private void expectPrefixMatch(IndexSearcher searcher, String term, long count) throws IOException {
         Query q = wildcardFieldType.fieldType().prefixQuery(term, null, MOCK_CONTEXT);
         TopDocs td = searcher.search(q, 10, Sort.RELEVANCE);
-        assertThat(td.totalHits.value, equalTo(count));
+        assertThat(td.totalHits.value(), equalTo(count));
     }
 
     public void testSearchResultsVersusKeywordField() throws IOException {
@@ -390,8 +391,8 @@ public void testSearchResultsVersusKeywordField() throws IOException {
             TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, values.size() + 1, Sort.RELEVANCE);
             assertThat(
                 keywordFieldQuery + "\n" + wildcardFieldQuery,
-                wildcardFieldTopDocs.totalHits.value,
-                equalTo(kwTopDocs.totalHits.value)
+                wildcardFieldTopDocs.totalHits.value(),
+                equalTo(kwTopDocs.totalHits.value())
             );
 
             HashSet<Integer> expectedDocs = new HashSet<>();
@@ -497,7 +498,7 @@ public void testRangeQueryVersusKeywordField() throws IOException {
 
             TopDocs kwTopDocs = searcher.search(keywordFieldQuery, 10, Sort.RELEVANCE);
             TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.RELEVANCE);
-            assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(kwTopDocs.totalHits.value));
+            assertThat(wildcardFieldTopDocs.totalHits.value(), equalTo(kwTopDocs.totalHits.value()));
 
             HashSet<Integer> expectedDocs = new HashSet<>();
             for (ScoreDoc topDoc : kwTopDocs.scoreDocs) {
@@ -642,7 +643,7 @@ public void testWildcardAcceleration() throws IOException, ParseException {
     public void testQueryCachingEquality() throws IOException, ParseException {
         String pattern = "A*b*B?a";
         // Case sensitivity matters when it comes to caching
-        Automaton caseSensitiveAutomaton = WildcardQuery.toAutomaton(new Term("field", pattern));
+        Automaton caseSensitiveAutomaton = WildcardQuery.toAutomaton(new Term("field", pattern), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
         Automaton caseInSensitiveAutomaton = AutomatonQueries.toCaseInsensitiveWildcardAutomaton(new Term("field", pattern));
         BinaryDvConfirmedAutomatonQuery csQ = new BinaryDvConfirmedAutomatonQuery(
             new MatchAllDocsQuery(),
@@ -660,7 +661,10 @@ public void testQueryCachingEquality() throws IOException, ParseException {
         assertNotEquals(csQ.hashCode(), ciQ.hashCode());
 
         // Same query should be equal
-        Automaton caseSensitiveAutomaton2 = WildcardQuery.toAutomaton(new Term("field", pattern));
+        Automaton caseSensitiveAutomaton2 = WildcardQuery.toAutomaton(
+            new Term("field", pattern),
+            Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
+        );
         BinaryDvConfirmedAutomatonQuery csQ2 = new BinaryDvConfirmedAutomatonQuery(
             new MatchAllDocsQuery(),
             "field",
@@ -880,11 +884,11 @@ private Query rewriteFiltersToMustsForComparisonPurposes(Query q) {
         if (q instanceof BooleanQuery bq) {
             BooleanQuery.Builder result = new BooleanQuery.Builder();
             for (BooleanClause cq : bq.clauses()) {
-                Query rewritten = rewriteFiltersToMustsForComparisonPurposes(cq.getQuery());
-                if (cq.getOccur() == Occur.FILTER) {
+                Query rewritten = rewriteFiltersToMustsForComparisonPurposes(cq.query());
+                if (cq.occur() == Occur.FILTER) {
                     result.add(rewritten, Occur.MUST);
                 } else {
-                    result.add(rewritten, cq.getOccur());
+                    result.add(rewritten, cq.occur());
                 }
             }
             return result.build();
@@ -1013,8 +1017,9 @@ protected String convertToRandomRegex(String randomValue) {
         }
 
         // Assert our randomly generated regex actually matches the provided raw input.
-        RegExp regex = new RegExp(result.toString());
-        Automaton automaton = regex.toAutomaton();
+        int includeDeprecatedComplement = RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT;
+        RegExp regex = new RegExp(result.toString(), includeDeprecatedComplement);
+        Automaton automaton = Operations.determinize(regex.toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
         ByteRunAutomaton bytesMatcher = new ByteRunAutomaton(automaton);
         BytesRef br = new BytesRef(randomValue);
         assertTrue(
diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java
index fe62d4e2d2639..0b40828b8e86c 100644
--- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java
+++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java
@@ -48,6 +48,7 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest
         .setting("xpack.security.autoconfiguration.enabled", "false")
         .user(USER, PASS)
         .feature(FeatureFlag.TIME_SERIES_MODE)
+        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .build();
 
     public CoreWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index c57e5653d1279..a56ddaabe8280 100644
--- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -435,6 +435,7 @@ public void testRollupAfterRestart() throws Exception {
 
             final Request bulkRequest = new Request("POST", "/_bulk");
             bulkRequest.setJsonEntity(bulk.toString());
+            bulkRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(fieldNamesFieldOk()));
             client().performRequest(bulkRequest);
 
             // create the rollup job
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
index 3e57faea848bf..dc9afb1bec237 100644
--- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
@@ -18,8 +18,6 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Strings;
-import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.test.rest.RestTestLegacyFeatures;
 import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus;
@@ -93,9 +91,6 @@ protected Settings restClientSettings() {
     }
 
     public void testDeploymentSurvivesRestart() throws Exception {
-        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // condition will always be true from v8, can be removed
-        var originalClusterSupportsNlpModels = oldClusterHasFeature(RestTestLegacyFeatures.ML_NLP_SUPPORTED);
-        assumeTrue("NLP model deployments added in 8.0", originalClusterSupportsNlpModels);
 
         String modelId = "trained-model-full-cluster-restart";
 
@@ -139,7 +134,7 @@ private void waitForDeploymentStarted(String modelId) throws Exception {
                 equalTo("fully_allocated")
             );
             assertThat(stat.toString(), XContentMapValues.extractValue("deployment_stats.state", stat), equalTo("started"));
-        }, 90, TimeUnit.SECONDS);
+        }, 120, TimeUnit.SECONDS);
     }
 
     private void assertInfer(String modelId) throws IOException {
diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
index d61c143098fcb..f502683e42eb2 100644
--- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
+++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
@@ -377,7 +377,7 @@ private void assertDocs(
         try {
             logger.info(searchResponse);
             // check hit count
-            assertEquals(numDocs, searchResponse.getHits().getTotalHits().value);
+            assertEquals(numDocs, searchResponse.getHits().getTotalHits().value());
             // check that _index is properly set
             assertTrue(Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getIndex).allMatch(index::equals));
             // check that all _ids are there
@@ -404,7 +404,7 @@ private void assertDocs(
         );
         try {
             logger.info(searchResponse);
-            assertEquals(1, searchResponse.getHits().getTotalHits().value);
+            assertEquals(1, searchResponse.getHits().getTotalHits().value());
             assertEquals(id, searchResponse.getHits().getHits()[0].getId());
             assertEquals(sourceForDoc(num), searchResponse.getHits().getHits()[0].getSourceAsString());
         } finally {
@@ -456,7 +456,7 @@ private void assertDocs(
                 );
                 try {
                     logger.info(searchResponse);
-                    assertEquals(typeCount, searchResponse.getHits().getTotalHits().value);
+                    assertEquals(typeCount, searchResponse.getHits().getTotalHits().value());
                     for (SearchHit hit : searchResponse.getHits().getHits()) {
                         DocumentField typeField = hit.field("_type");
                         assertNotNull(typeField);
@@ -482,7 +482,7 @@ private void assertDocs(
             );
             try {
                 logger.info(searchResponse);
-                assertEquals(0, searchResponse.getHits().getTotalHits().value);
+                assertEquals(0, searchResponse.getHits().getTotalHits().value());
                 assertEquals(numberOfShards, searchResponse.getSuccessfulShards());
                 // When all shards are skipped, at least one of them is queried in order to provide a proper search response.
                 assertEquals(numberOfShards - 1, searchResponse.getSkippedShards());
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java
index 650779cfbc85d..ea1b2cdac5a1f 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.SECURITY_ROLE_DESCRIPTION;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomApplicationPrivileges;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomIndicesPrivileges;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomManageRolesPrivileges;
@@ -43,8 +44,8 @@ public class RolesBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
     public void testRolesWithDescription() throws Exception {
         assumeTrue(
-            "The role description is supported after transport version: " + TransportVersions.SECURITY_ROLE_DESCRIPTION,
-            minimumTransportVersion().before(TransportVersions.SECURITY_ROLE_DESCRIPTION)
+            "The role description is supported after transport version: " + SECURITY_ROLE_DESCRIPTION,
+            minimumTransportVersion().before(SECURITY_ROLE_DESCRIPTION)
         );
         switch (CLUSTER_TYPE) {
             case OLD -> {
@@ -75,7 +76,7 @@ public void testRolesWithDescription() throws Exception {
             }
             case MIXED -> {
                 try {
-                    this.createClientsByVersion(TransportVersions.SECURITY_ROLE_DESCRIPTION);
+                    this.createClientsByVersion(SECURITY_ROLE_DESCRIPTION);
                     // succeed when role description is not provided
                     final String initialRole = randomRoleDescriptorSerialized();
                     createRole(client(), "my-valid-mixed-role", initialRole);
@@ -116,7 +117,7 @@ public void testRolesWithDescription() throws Exception {
                             e.getMessage(),
                             containsString(
                                 "all nodes must have version ["
-                                    + TransportVersions.SECURITY_ROLE_DESCRIPTION.toReleaseVersion()
+                                    + SECURITY_ROLE_DESCRIPTION.toReleaseVersion()
                                     + "] or higher to support specifying role description"
                             )
                         );
@@ -130,7 +131,7 @@ public void testRolesWithDescription() throws Exception {
                             e.getMessage(),
                             containsString(
                                 "all nodes must have version ["
-                                    + TransportVersions.SECURITY_ROLE_DESCRIPTION.toReleaseVersion()
+                                    + SECURITY_ROLE_DESCRIPTION.toReleaseVersion()
                                     + "] or higher to support specifying role description"
                             )
                         );
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
index dddba9b7b0fba..02dc679152bf4 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
@@ -445,7 +445,7 @@ private List<String> getAllTokenIds() throws IOException {
             final SearchHits searchHits = response.getHits();
             assertThat(
                 "Search request used with size parameter that was too small to fetch all tokens.",
-                searchHits.getTotalHits().value,
+                searchHits.getTotalHits().value(),
                 lessThanOrEqualTo(searchSize)
             );
             final List<String> tokenIds = Arrays.stream(searchHits.getHits()).map(searchHit -> {
diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle
index 43d6d9463e0d1..986baf867b501 100644
--- a/x-pack/qa/runtime-fields/build.gradle
+++ b/x-pack/qa/runtime-fields/build.gradle
@@ -44,6 +44,7 @@ subprojects {
       setting 'xpack.security.enabled', 'false'
 
       requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+      requiresFeature 'es.sub_objects_auto_feature_flag_enabled', Version.fromString("8.16.0")
     }
 
     tasks.named("yamlRestTest").configure {