Merge branch 'main' into markjhoy/add-sparse_vector_index_options-support_semantic_text

Author: markjhoy

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.9", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -302,8 +302,8 @@ steps:
         env:
           BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.10 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.10
+      - label: "{{matrix.image}} / 8.17.9 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.9
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -316,7 +316,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.10
+          BWC_VERSION: 8.17.9
 
       - label: "{{matrix.image}} / 8.18.5 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.5
@@ -334,8 +334,8 @@ steps:
         env:
           BWC_VERSION: 8.18.5
 
-      - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
+      - label: "{{matrix.image}} / 8.19.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,7 +348,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.0
+          BWC_VERSION: 8.19.1
 
       - label: "{{matrix.image}} / 9.0.5 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.5

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.10 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.10#bwcTest
+      - label: 8.17.9 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.9#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.10
+          BWC_VERSION: 8.17.9
         retry:
           automatic:
             - exit_status: "-1"
@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.0#bwcTest
+      - label: 8.19.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.0
+          BWC_VERSION: 8.19.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.9", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.9", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.ci/bwcVersions

@@ -16,9 +16,9 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.6"
-  - "8.17.10"
+  - "8.17.9"
   - "8.18.5"
-  - "8.19.0"
+  - "8.19.1"
   - "9.0.5"
   - "9.1.1"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,7 @@
 BWC_VERSION:
-  - "8.17.10"
+  - "8.17.9"
   - "8.18.5"
-  - "8.19.0"
+  - "8.19.1"
   - "9.0.5"
   - "9.1.1"
   - "9.2.0"

docs/changelog/132088.yaml

@@ -0,0 +1,5 @@
+pr: 132088
+summary: Fix combine result for `ingest_took`
+area: ES|QL
+type: bug
+issues: []

muted-tests.yml

@@ -290,7 +290,7 @@ tests:
   method: testOneRemoteClusterPartial
   issue: https://github.com/elastic/elasticsearch/issues/124055
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyOnTheLookupIndex ASYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyOnTheLookupIndex}
   issue: https://github.com/elastic/elasticsearch/issues/128030
 - class: org.elasticsearch.packaging.test.EnrollmentProcessTests
   method: test20DockerAutoFormCluster
@@ -314,11 +314,8 @@ tests:
   method: testEarlyTermination
   issue: https://github.com/elastic/elasticsearch/issues/128721
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
-  method: test {lookup-join.EnrichLookupStatsBug ASYNC}
+  method: test {csv-spec:lookup-join.EnrichLookupStatsBug}
   issue: https://github.com/elastic/elasticsearch/issues/129228
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
-  method: test {lookup-join.EnrichLookupStatsBug SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/129229
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {lookup-join.MultipleBatches*
   issue: https://github.com/elastic/elasticsearch/issues/129210
@@ -363,7 +360,7 @@ tests:
   method: testWriteCanRejectOnPrimaryBasedOnMaxOperationSize
   issue: https://github.com/elastic/elasticsearch/issues/130281
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyOnFrom SYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyOnFrom}
   issue: https://github.com/elastic/elasticsearch/issues/130296
 - class: org.elasticsearch.xpack.esql.inference.bulk.BulkInferenceExecutorTests
   method: testSuccessfulExecution
@@ -378,7 +375,7 @@ tests:
   method: failed tasks output logged to console when spooling true
   issue: https://github.com/elastic/elasticsearch/issues/119509
 - class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyFromRow ASYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyFromRow}
   issue: https://github.com/elastic/elasticsearch/issues/130642
 - class: org.elasticsearch.indices.stats.IndexStatsIT
   method: testFilterCacheStats
@@ -390,7 +387,7 @@ tests:
   method: testOpenContextsAfterRejections
   issue: https://github.com/elastic/elasticsearch/issues/130821
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyOnFromAfterStats ASYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyOnFromAfterStats}
   issue: https://github.com/elastic/elasticsearch/issues/131148
 - class: org.elasticsearch.xpack.esql.ccq.MultiClustersIT
   method: testLookupJoinAliases
@@ -530,6 +527,12 @@ tests:
 - class: org.elasticsearch.xpack.sql.qa.mixed_node.SqlCompatIT
   method: testNullsOrderWithMissingOrderSupportQueryingNewNode
   issue: https://github.com/elastic/elasticsearch/issues/132249
+- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/132273
+- class: org.elasticsearch.common.logging.JULBridgeTests
+  method: testThrowable
+  issue: https://github.com/elastic/elasticsearch/issues/132280
 
 # Examples:
 #
@@ -561,11 +564,10 @@ tests:
 #    issue: "https://github.com/elastic/elasticsearch/..."
 #  Note that this mutes for the unit-test-like CsvTests only.
 #  Muting all the integration tests can be done using the class "org.elasticsearch.xpack.esql.**".
-#  Consider however, that some tests are named as "test {file.test SYNC}" and "ASYNC" in the integration tests.
 #  To mute all 3 tests safely everywhere use:
 #  - class: "org.elasticsearch.xpack.esql.**"
 #    method: "test {union_types.MultiIndexIpStringStatsInline}"
 #    issue: "https://github.com/elastic/elasticsearch/..."
 #  - class: "org.elasticsearch.xpack.esql.**"
 #    method: "test {union_types.MultiIndexIpStringStatsInline *}"
-#    issue: "https://github.com/elastic/elasticsearch/..."
+#    issue: "https://github.com/elastic/elasticsearch/..."

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -213,6 +213,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN_8_19 = def(8_841_0_62);
     public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_8_19 = def(8_841_0_63);
     public static final TransportVersion ESQL_FIXED_INDEX_LIKE_8_19 = def(8_841_0_64);
+    public static final TransportVersion INITIAL_ELASTICSEARCH_8_19_1 = def(8_841_0_65);
     public static final TransportVersion V_9_0_0 = def(9_000_0_09);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);

server/src/main/java/org/elasticsearch/Version.java

@@ -205,14 +205,14 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_17_7 = new Version(8_17_07_99);
     public static final Version V_8_17_8 = new Version(8_17_08_99);
     public static final Version V_8_17_9 = new Version(8_17_09_99);
-    public static final Version V_8_17_10 = new Version(8_17_10_99);
     public static final Version V_8_18_0 = new Version(8_18_00_99);
     public static final Version V_8_18_1 = new Version(8_18_01_99);
     public static final Version V_8_18_2 = new Version(8_18_02_99);
     public static final Version V_8_18_3 = new Version(8_18_03_99);
     public static final Version V_8_18_4 = new Version(8_18_04_99);
     public static final Version V_8_18_5 = new Version(8_18_05_99);
     public static final Version V_8_19_0 = new Version(8_19_00_99);
+    public static final Version V_8_19_1 = new Version(8_19_01_99);
     public static final Version V_9_0_0 = new Version(9_00_00_99);
     public static final Version V_9_0_1 = new Version(9_00_01_99);
     public static final Version V_9_0_2 = new Version(9_00_02_99);

server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.Iterator;
+import java.util.List;
 
 /**
  * A response of a bulk execution. Holding a response for each item responding (in order) of the
@@ -166,4 +167,32 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
             return builder.startArray(ITEMS);
         }), Iterators.forArray(responses), Iterators.<ToXContent>single((builder, p) -> builder.endArray().endObject()));
     }
+
+    /**
+     * Combine many bulk responses into one.
+     */
+    public static BulkResponse combine(List<BulkResponse> responses) {
+        long tookInMillis = 0;
+        long ingestTookInMillis = NO_INGEST_TOOK;
+        int itemResponseCount = 0;
+        for (BulkResponse response : responses) {
+            tookInMillis += response.getTookInMillis();
+            if (response.getIngestTookInMillis() != NO_INGEST_TOOK) {
+                if (ingestTookInMillis == NO_INGEST_TOOK) {
+                    ingestTookInMillis = 0;
+                }
+                ingestTookInMillis += response.getIngestTookInMillis();
+            }
+            itemResponseCount += response.getItems().length;
+        }
+        BulkItemResponse[] bulkItemResponses = new BulkItemResponse[itemResponseCount];
+        int i = 0;
+        for (BulkResponse response : responses) {
+            for (BulkItemResponse itemResponse : response.getItems()) {
+                bulkItemResponses[i++] = itemResponse;
+            }
+        }
+
+        return new BulkResponse(bulkItemResponses, tookInMillis, ingestTookInMillis);
+    }
 }