Merge branch 'main' into enhancement/unmapped_fields_rewrite

Author: javanna

.buildkite/pipelines/pull-request-performance-benchmark.yml

@@ -0,0 +1,3 @@
+steps:
+  - label: ":pipeline: Generate steps"
+    command: bash .buildkite/scripts/generate-pr-performance-benchmark.sh | buildkite-agent pipeline upload

.buildkite/pipelines/pull-request/performance-benchmark.yml

@@ -27,7 +27,8 @@
       "set_commit_status": false,
       "build_on_commit": false,
       "build_on_comment": true,
-      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
+      "target_branch": "main",
+      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\S+)( please)?$"
     }
   ]
 }

.buildkite/pull-requests.json

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
+env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
+merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
+
+buildkite-agent meta-data set pr_comment:custom-body:body \
+  "This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+buildkite-agent meta-data set pr_comment:custom-baseline:head \
+  "* Baseline: ${merge_base} (env ID ${env_id_baseline})"
+buildkite-agent meta-data set pr_comment:custom-contender:head \
+  "* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
+
+cat << _EOF_
+steps:
+  - label: Trigger baseline benchmark
+    trigger: elasticsearch-performance-esbench-pr
+    build:
+      message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
+      branch: master
+      env:
+        CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
+        ENV_ID: ${env_id_baseline}
+        REVISION: ${merge_base}
+  - label: Trigger contender benchmark
+    trigger: elasticsearch-performance-esbench-pr
+    build:
+      message: Contender benchmark for PR${GITHUB_PR_NUMBER}
+      branch: master
+      env:
+        CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
+        ENV_ID: ${env_id_contender}
+        ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
+        REVISION: ${GITHUB_PR_TRIGGERED_SHA}
+  - wait: ~
+  - label: Modify PR comment
+    command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
+_EOF_

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -323,7 +323,7 @@ spec:
       name: elasticsearch / pull-request / performance-benchmark
     spec:
       repository: elastic/elasticsearch
-      pipeline_file: .buildkite/pipelines/pull-request/performance-benchmark.yml
+      pipeline_file: .buildkite/pipelines/pull-request-performance-benchmark.yml
       env:
         ELASTIC_PR_COMMENTS_ENABLED: "true"
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"

catalog-info.yaml

@@ -0,0 +1,16 @@
+pr: 130325
+summary: "Simulate API: Return 400 on invalid processor(s)"
+area: Ingest Node
+type: bug
+issues:
+ - 120731
+breaking:
+  title: Return 400 on invalid processor(s) in Simulate API
+  area: Ingest
+  details: "In earlier versions of {es}, the Simulate API would return a 500 error\
+    \ when encountering invalid processors. Now, it returns a 400 Bad Request error\
+    \ instead."
+  impact: Callers should expect a 400 Bad Request response when the Simulate API encounters
+    invalid processors. This change improves error handling and provides clearer feedback
+    on request issues.
+  notable: false

docs/changelog/130325.yaml

@@ -0,0 +1,6 @@
+pr: 74274
+summary: Introduce new rescorer based on script
+area: Search
+type: feature
+issues:
+  - 52338

docs/changelog/74274.yaml

@@ -28,7 +28,7 @@ dependencies {
 
 restResources {
   restApi {
-    include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search'
+    include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search', 'simulate'
   }
 }
 

modules/ingest-common/build.gradle

@@ -154,3 +154,48 @@ teardown:
       ingest.processor_grok: {}
   - length: { patterns: 318 }
   - match: { patterns.PATH: "(?:%{UNIXPATH}|%{WINPATH})" }
+
+
+---
+"Test simulate with invalid GROK pattern":
+  - requires:
+        cluster_features: [ "simulate.ingest.400_on_failure" ]
+        reason: "simulate.ingest returned 500 on failure before"
+  - skip:
+        features: headers
+  - do:
+      catch: bad_request
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        pipeline: "invalid-grok"
+        body: >
+          {
+            "docs": [
+              {
+                "_index": "index-1",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ],
+            "pipeline_substitutions": {
+              "invalid-grok": {
+                "description": "invalid grok pattern",
+                "processors": [
+                  {
+                    "grok": {
+                      "field": "field",
+                      "patterns": [
+                        "%{INVALID_PATTERN:field}"
+                      ]
+                    }
+                  }
+                ]
+              }
+            }
+          }
+  - match: { status: 400 }
+  - match: { error.reason: "[patterns] Invalid regex pattern found in: [%{INVALID_PATTERN:field}]. Unable to find pattern [INVALID_PATTERN] in Grok's pattern dictionary" }
+  - match: { error.property_name: "patterns" }
+  - match: { error.processor_type: "grok" }

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/120_grok.yml

@@ -299,12 +299,6 @@ tests:
 - class: org.elasticsearch.compute.operator.LimitOperatorTests
   method: testEarlyTermination
   issue: https://github.com/elastic/elasticsearch/issues/128721
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
-  method: test {csv-spec:lookup-join.EnrichLookupStatsBug}
-  issue: https://github.com/elastic/elasticsearch/issues/129228
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
-  method: test {lookup-join.MultipleBatches*
-  issue: https://github.com/elastic/elasticsearch/issues/129210
 - class: org.elasticsearch.entitlement.runtime.policy.FileAccessTreeTests
   method: testWindowsMixedCaseAccess
   issue: https://github.com/elastic/elasticsearch/issues/129167
@@ -471,9 +465,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.RevertModelSnapshotIT
   method: testRevertModelSnapshot
   issue: https://github.com/elastic/elasticsearch/issues/132733
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
-  method: test {csv-spec:lookup-join.MvJoinKeyFromRowExpanded}
-  issue: https://github.com/elastic/elasticsearch/issues/132778
 - class: org.elasticsearch.gradle.internal.transport.TransportVersionManagementPluginFuncTest
   method: definitions have primary ids which cannot change
   issue: https://github.com/elastic/elasticsearch/issues/132788
@@ -615,6 +606,15 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=search.vectors/100_knn_nested_search/nested kNN search inner_hits & profiling}
   issue: https://github.com/elastic/elasticsearch/issues/133273
+- class: org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapperTests
+  method: testDefaultIndexOptions {p0=true}
+  issue: https://github.com/elastic/elasticsearch/issues/133326
+- class: org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapperTests
+  method: testDefaultIndexOptions {p0=false}
+  issue: https://github.com/elastic/elasticsearch/issues/133327
+- class: org.elasticsearch.xpack.security.authc.AuthenticationServiceTests
+  method: testInvalidToken
+  issue: https://github.com/elastic/elasticsearch/issues/133328
 
 # Examples:
 #