Merge remote-tracking branch 'upstream/main' into invalid-grok-500

* upstream/main: (32 commits)
  Speed up loading keyword fields with index sorts (#132950)
  Mute org.elasticsearch.index.mapper.LongFieldMapperTests testSyntheticSourceWithTranslogSnapshot #132964
  Simplify EsqlSession (#132848)
  Implement WriteLoadConstraintDecider#canAllocate (#132041)
  Mute org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT test {p0=search/400_synthetic_source/_doc_count} #132965
  Switch to PR-based benchmark pipeline defined in ES repo (#132941)
  Breakdown undesired allocations by shard routing role (#132235)
  Implement v_magnitude function (#132765)
  Introduce execution location marker for better handling of remote/local compatibility (#132205)
  Mute org.elasticsearch.cluster.ClusterInfoServiceIT testMaxQueueLatenciesInClusterInfo #132957
  Unmuting simulate index data stream mapping overrides yaml rest test (#132946)
  Remove CrossClusterCancellationIT.createLocalIndex() (#132952)
  Mute org.elasticsearch.index.mapper.LongFieldMapperTests testFetch #132956
  Fix failing UT by adding a required capability (#132947)
  Precompute the BitsetCacheKey hashCode (#132875)
  Adding simulate ingest effective mapping (#132833)
  Mute org.elasticsearch.index.mapper.LongFieldMapperTests testFetchMany #132948
  Rename skipping logic to remove hard link to skip_unavailable (#132861)
  Store ignored source in unique stored fields per entry (#132142)
  Add random tests with match_only_text multi-field (#132380)
  ...

Author: szybia

.buildkite/pipelines/pull-request/performance-benchmark.yml

@@ -0,0 +1,3 @@
+steps:
+  - label: ":pipeline: TODO"
+    command: echo TODO

.buildkite/pull-requests.json

@@ -18,7 +18,7 @@
     },
     {
       "enabled": true,
-      "pipeline_slug": "elasticsearch-performance-esbench-pr",
+      "pipeline_slug": "elasticsearch-pull-request-performance-benchmark",
       "allow_org_users": true,
       "allowed_repo_permissions": [
         "admin",

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramGenerationBench.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.benchmark.exponentialhistogram;
 
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -59,7 +60,7 @@ public class ExponentialHistogramGenerationBench {
     @Setup
     public void setUp() {
         random = ThreadLocalRandom.current();
-        histoGenerator = new ExponentialHistogramGenerator(bucketCount);
+        histoGenerator = ExponentialHistogramGenerator.create(bucketCount, ExponentialHistogramCircuitBreaker.noop());
 
         DoubleSupplier nextRandom = () -> distribution.equals("GAUSSIAN") ? random.nextGaussian() : random.nextDouble();
 

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.exponentialhistogram.BucketIterator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -56,13 +57,14 @@ public class ExponentialHistogramMergeBench {
     @Setup
     public void setUp() {
         random = ThreadLocalRandom.current();
-        histoMerger = new ExponentialHistogramMerger(bucketCount);
+        ExponentialHistogramCircuitBreaker breaker = ExponentialHistogramCircuitBreaker.noop();
+        histoMerger = ExponentialHistogramMerger.create(bucketCount, breaker);
 
-        ExponentialHistogramGenerator initial = new ExponentialHistogramGenerator(bucketCount);
+        ExponentialHistogramGenerator initialGenerator = ExponentialHistogramGenerator.create(bucketCount, breaker);
         for (int j = 0; j < bucketCount; j++) {
-            initial.add(Math.pow(1.001, j));
+            initialGenerator.add(Math.pow(1.001, j));
         }
-        ExponentialHistogram initialHisto = initial.get();
+        ExponentialHistogram initialHisto = initialGenerator.getAndClear();
         int cnt = getBucketCount(initialHisto);
         if (cnt < bucketCount) {
             throw new IllegalArgumentException("Expected bucket count to be " + bucketCount + ", but was " + cnt);
@@ -72,14 +74,14 @@ public void setUp() {
         int dataPointSize = (int) Math.round(bucketCount * mergedHistoSizeFactor);
 
         for (int i = 0; i < toMerge.length; i++) {
-            ExponentialHistogramGenerator generator = new ExponentialHistogramGenerator(dataPointSize);
+            ExponentialHistogramGenerator generator = ExponentialHistogramGenerator.create(dataPointSize, breaker);
 
             int bucketIndex = 0;
             for (int j = 0; j < dataPointSize; j++) {
                 bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
                 generator.add(Math.pow(1.001, bucketIndex));
             }
-            toMerge[i] = generator.get();
+            toMerge[i] = generator.getAndClear();
             cnt = getBucketCount(toMerge[i]);
             if (cnt < dataPointSize) {
                 throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java

@@ -82,4 +82,13 @@ public void packAsBinaryLegacy(Blackhole bh) {
             bh.consume(packed);
         }
     }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void packAsBinaryPanama(Blackhole bh) {
+        for (int i = 0; i < numVectors; i++) {
+            BQVectorUtils.packAsBinary(qVectors[i], packed);
+            bh.consume(packed);
+        }
+    }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java

@@ -83,4 +83,13 @@ public void transposeHalfByteLegacy(Blackhole bh) {
             bh.consume(packed);
         }
     }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void transposeHalfBytePanama(Blackhole bh) {
+        for (int i = 0; i < numVectors; i++) {
+            BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
+            bh.consume(packed);
+        }
+    }
 }

catalog-info.yaml

@@ -301,3 +301,39 @@ spec:
         Daily:
           branch: main
           cronline: "@daily"
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-pull-request-performance-benchmark
+  description: Elasticsearch pull request performance benchmark
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-pull-request-performance-benchmark
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: ":elasticsearch: Runs performance benchmark in the PR context"
+      name: elasticsearch / pull-request / performance-benchmark
+    spec:
+      repository: elastic/elasticsearch
+      pipeline_file: .buildkite/pipelines/pull-request/performance-benchmark.yml
+      env:
+        ELASTIC_PR_COMMENTS_ENABLED: "true"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
+        SLACK_NOTIFICATIONS_CHANNEL: "#es-perf-build"
+        SLACK_NOTIFICATIONS_ALL_BRANCHES: "false" # TODO: change to true later
+      teams:
+        elasticsearch-team: {}
+        ml-core: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        build_pull_requests: true
+        trigger_mode: none

docs/changelog/132675.yaml

@@ -0,0 +1,5 @@
+pr: 132675
+summary: Add second max queue latency stat to `ClusterInfo`
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/132765.yaml

@@ -0,0 +1,5 @@
+pr: 132765
+summary: Implement `v_magnitude` function
+area: ES|QL
+type: feature
+issues: [132768]

docs/changelog/132833.yaml

@@ -0,0 +1,5 @@
+pr: 132833
+summary: Adding simulate ingest effective mapping
+area: Ingest Node
+type: enhancement
+issues: []