Merge branch 'main' into fix/testApmIntegration-2

Author: ldematte

.ci/scripts/resolve-dra-manifest.sh

@@ -6,7 +6,8 @@ strip_version() {
 }
 
 fetch_build() {
-  curl -sS https://artifacts-$1.elastic.co/$2/latest/$3.json \
+  >&2 echo "Checking for build id: https://artifacts-$1.elastic.co/$2/latest/$3.json"
+  curl -sSf https://artifacts-$1.elastic.co/$2/latest/$3.json \
     | jq -r '.build_id'
 }
 
@@ -15,7 +16,15 @@ BRANCH="${BRANCH:-$2}"
 ES_VERSION="${ES_VERSION:-$3}"
 WORKFLOW=${WORKFLOW:-$4}
 
-LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
+if [[ "$WORKFLOW" == "staging" ]]; then
+  LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $ES_VERSION)
+elif [[ "$WORKFLOW" == "snapshot" ]]; then
+  LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
+else
+  echo "Unknown workflow: $WORKFLOW"
+  exit 1
+fi
+
 LATEST_VERSION=$(strip_version $LATEST_BUILD)
 
 # If the latest artifact version doesn't match what we expect, try the corresponding version branch.

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.lucene.LuceneSourceOperator;
+import org.elasticsearch.compute.lucene.ShardRefCounted;
 import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
 import org.elasticsearch.compute.operator.topn.TopNOperator;
 import org.elasticsearch.core.IOUtils;
@@ -477,6 +478,7 @@ private void setupPages() {
                         pages.add(
                             new Page(
                                 new DocVector(
+                                    ShardRefCounted.ALWAYS_REFERENCED,
                                     blockFactory.newConstantIntBlockWith(0, end - begin).asVector(),
                                     blockFactory.newConstantIntBlockWith(ctx.ord, end - begin).asVector(),
                                     docs.build(),
@@ -512,7 +514,14 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                         if (size >= BLOCK_LENGTH) {
                             pages.add(
                                 new Page(
-                                    new DocVector(blockFactory.newConstantIntVector(0, size), leafs.build(), docs.build(), null).asBlock()
+                                    new DocVector(
+
+                                        ShardRefCounted.ALWAYS_REFERENCED,
+                                        blockFactory.newConstantIntVector(0, size),
+                                        leafs.build(),
+                                        docs.build(),
+                                        null
+                                    ).asBlock()
                                 )
                             );
                             docs = blockFactory.newIntVectorBuilder(BLOCK_LENGTH);
@@ -525,6 +534,8 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                     pages.add(
                         new Page(
                             new DocVector(
+
+                                ShardRefCounted.ALWAYS_REFERENCED,
                                 blockFactory.newConstantIntBlockWith(0, size).asVector(),
                                 leafs.build().asBlock().asVector(),
                                 docs.build(),
@@ -551,6 +562,8 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                         pages.add(
                             new Page(
                                 new DocVector(
+
+                                    ShardRefCounted.ALWAYS_REFERENCED,
                                     blockFactory.newConstantIntVector(0, 1),
                                     blockFactory.newConstantIntVector(next.ord, 1),
                                     blockFactory.newConstantIntVector(next.itr.nextInt(), 1),

docs/changelog/122497.yaml

@@ -0,0 +1,5 @@
+pr: 122497
+summary: Check if index patterns conform to valid format before validation
+area: CCS
+type: enhancement
+issues: []

docs/changelog/129370.yaml

@@ -0,0 +1,7 @@
+pr: 129370
+summary: Avoid dropping aggregate groupings in local plans
+area: ES|QL
+type: bug
+issues:
+ - 129811
+ - 128054

docs/changelog/129454.yaml

@@ -0,0 +1,5 @@
+pr: 129454
+summary: Aggressive release of shard contexts
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/130032.yaml

@@ -0,0 +1,12 @@
+pr: 130032
+summary: ES|QL cross-cluster querying is now generally available
+area: ES|QL
+type: feature
+issues: []
+highlight:
+  title: ES|QL cross-cluster querying is now generally available
+  body: |-
+    The ES|QL Cross-Cluster querying feature has been in technical preview since 8.13.
+    As of releases 8.19.0 and 9.1.0 this is now generally available.
+    This feature allows you to run ES|QL queries across multiple clusters.
+  notable: true

docs/changelog/130083.yaml

@@ -0,0 +1,5 @@
+pr: 130083
+summary: Fix timeout bug in DBQ deletion of unused and orphan ML data
+area: Machine Learning
+type: bug
+issues: []

docs/reference/elasticsearch/index-settings/slow-log.md

@@ -8,7 +8,7 @@ navigation_title: Slow log
 
 The slow log records database searching and indexing events that have execution durations above specified thresholds. You can use these logs to investigate analyze or troubleshoot your cluster’s historical search and indexing performance.
 
-Slow logs report task duration at the shard level for searches, and at the index level for indexing, but might not encompass the full task execution time observed on the client. For example, slow logs don’t surface HTTP network delays or the impact of [task queues](docs-content://troubleshoot/elasticsearch/task-queue-backlog.md).
+Slow logs report task duration at the shard level for searches, and at the index level for indexing, but might not encompass the full task execution time observed on the client. For example, slow logs don’t surface HTTP network delays or the impact of [task queues](docs-content://troubleshoot/elasticsearch/task-queue-backlog.md). For more information about the higher-level operations affecting response times, refer to [Reading and writing documents](docs-content://deploy-manage/distributed-architecture/reading-and-writing-documents.md).
 
 Events that meet the specified threshold are emitted into [{{es}} logging](docs-content://deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md) under the `fileset.name` of `slowlog`. These logs can be viewed in the following locations:
 

gradle/verification-metadata.xml

@@ -1153,16 +1153,16 @@
             <sha256 value="88ac9fd1bb51f82bcc664cc1eb9c225c90dc4389d660231b4cc737bebfe7d0aa" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.squareup.okhttp3" name="okhttp" version="4.11.0">
-         <artifact name="okhttp-4.11.0.jar">
-            <sha256 value="ee8f6bd6cd1257013d748330f4ca147638a9fbcb52fb388d5ac93cf53408745d" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.squareup.okhttp3" name="logging-interceptor" version="4.12.0">
          <artifact name="logging-interceptor-4.12.0.jar">
             <sha256 value="f3e8d5f0903c250c2b55d2f47fcfe008e80634385da8385161c7a63aaed0c74c" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.squareup.okhttp3" name="okhttp" version="4.11.0">
+         <artifact name="okhttp-4.11.0.jar">
+            <sha256 value="ee8f6bd6cd1257013d748330f4ca147638a9fbcb52fb388d5ac93cf53408745d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.squareup.okhttp3" name="okhttp" version="4.12.0">
          <artifact name="okhttp-4.12.0.jar">
             <sha256 value="b1050081b14bb7a3a7e55a4d3ef01b5dcfabc453b4573a4fc019767191d5f4e0" origin="Generated by Gradle"/>
@@ -4601,6 +4601,11 @@
             <sha256 value="8cadd43ac5eb6d09de05faecca38b917a040bb9139c7edeb4cc81c740b713281" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm" version="9.8">
+         <artifact name="asm-9.8.jar">
+            <sha256 value="876eab6a83daecad5ca67eb9fcabb063c97b5aeb8cf1fca7a989ecde17522051" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm-analysis" version="7.2">
          <artifact name="asm-analysis-7.2.jar">
             <sha256 value="be922aae60ff1ff1768e8e6544a38a7f92bd0a6d6b0b9791f94955d1bd453de2" origin="Generated by Gradle"/>
@@ -4621,6 +4626,11 @@
             <sha256 value="85b29371884ba31bb76edf22323c2c24e172c3267a67152eba3d1ccc2e041ef2" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm-analysis" version="9.8">
+         <artifact name="asm-analysis-9.8.jar">
+            <sha256 value="e640732fbcd3c6271925a504f125e38384688f4dfbbf92c8622dfcee0d09edb9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm-commons" version="7.2">
          <artifact name="asm-commons-7.2.jar">
             <sha256 value="0e86b8b179c5fb223d1a880a0ff4960b6978223984b94e62e71135f2d8ea3558" origin="Generated by Gradle"/>
@@ -4686,6 +4696,11 @@
             <sha256 value="9929881f59eb6b840e86d54570c77b59ce721d104e6dfd7a40978991c2d3b41f" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm-tree" version="9.8">
+         <artifact name="asm-tree-9.8.jar">
+            <sha256 value="14b7880cb7c85eed101e2710432fc3ffb83275532a6a894dc4c4095d49ad59f1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm-util" version="7.2">
          <artifact name="asm-util-7.2.jar">
             <sha256 value="6e24913b021ffacfe8e7e053d6e0ccc731941148cfa078d4f1ed3d96904530f8" origin="Generated by Gradle"/>
@@ -4696,6 +4711,11 @@
             <sha256 value="f885be71b5c90556f5f1ad1c4f9276b29b96057c497d46666fe4ddbec3cb43c6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm-util" version="9.8">
+         <artifact name="asm-util-9.8.jar">
+            <sha256 value="8ba0460ecb28fd0e2980e5f3ef3433a513a457bc077f81a53bdc75b587a08d15" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.reactivestreams" name="reactive-streams" version="1.0.4">
          <artifact name="reactive-streams-1.0.4.jar">
             <sha256 value="f75ca597789b3dac58f61857b9ac2e1034a68fa672db35055a8fb4509e325f28" origin="Generated by Gradle"/>

libs/core/src/main/java/org/elasticsearch/core/Releasables.java

@@ -202,6 +202,11 @@ public boolean equals(Object obj) {
         }
     }
 
+    /** Creates a {@link Releasable} that calls {@link RefCounted#decRef()} when closed. */
+    public static Releasable fromRefCounted(RefCounted refCounted) {
+        return () -> refCounted.decRef();
+    }
+
     private static class ReleaseOnce extends AtomicReference<Releasable> implements Releasable {
         ReleaseOnce(Releasable releasable) {
             super(releasable);