Merge branch 'main' into unmute-ip-tests

Author: felixbarny

.gitignore

@@ -1,4 +1,7 @@
 
+# claude
+.claude
+
 # intellij files
 .idea/
 *.iml

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -167,6 +167,11 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                     }
                 }
             }
+
+            tasks.named('assemble').configure {
+                dependsOn buildTar
+            }
+
             artifacts {
                 it.add("default", buildTar)
                 it.add("extracted", buildExpanded)

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java

@@ -74,6 +74,7 @@ private void registerAndConfigureDistributionArchivesExtension(Project project)
             project.project(subProjectName, sub -> {
                 sub.getPlugins().apply(BasePlugin.class);
                 sub.getArtifacts().add(DEFAULT_CONFIGURATION_NAME, distributionArchive.getArchiveTask());
+                sub.getTasks().named("assemble").configure(task -> task.dependsOn(distributionArchive.getArchiveTask()));
                 var extractedConfiguration = sub.getConfigurations().create(EXTRACTED_CONFIGURATION_NAME);
                 extractedConfiguration.setCanBeResolved(false);
                 extractedConfiguration.setCanBeConsumed(true);

build-tools-internal/version.properties

@@ -22,7 +22,7 @@ reactive_streams    = 1.0.4
 
 antlr4            = 4.13.1
 # bouncy castle version for non-fips. fips jars use a different version
-bouncycastle=1.78.1
+bouncycastle=1.79
 # used by security and idp (need to be in sync due to cross-dependency in testing)
 opensaml = 4.3.0
 

distribution/packages/build.gradle

@@ -354,6 +354,10 @@ tasks.register('buildDeb', Deb) {
   configure(commonDebConfig('x64'))
 }
 
+tasks.named('assemble'){
+  dependsOn 'buildDeb', 'buildAarch64Deb'
+}
+
 Closure commonRpmConfig(String architecture) {
   return {
     configure(commonPackageConfig('rpm', architecture))
@@ -387,6 +391,11 @@ tasks.register('buildRpm', Rpm) {
   configure(commonRpmConfig('x64'))
 }
 
+tasks.named('assemble'){
+  dependsOn 'buildRpm', 'buildAarch64Rpm'
+}
+
+
 Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
 Closure rpmExists = { it -> new File('/bin/rpm').exists() || new File('/usr/bin/rpm').exists() || new File('/usr/local/bin/rpm').exists() }
 

docs/changelog/129693.yaml

@@ -0,0 +1,5 @@
+pr: 129693
+summary: Add top level normalizer for linear retriever
+area: Search
+type: enhancement
+issues: []

docs/changelog/130325.yaml

@@ -0,0 +1,16 @@
+pr: 130325
+summary: "Simulate API: Return 400 on invalid processor(s)"
+area: Ingest Node
+type: bug
+issues:
+ - 120731
+breaking:
+  title: Return 400 on invalid processor(s) in Simulate API
+  area: Ingest
+  details: "In earlier versions of {es}, the Simulate API would return a 500 error\
+    \ when encountering invalid processors. Now, it returns a 400 Bad Request error\
+    \ instead."
+  impact: Callers should expect a 400 Bad Request response when the Simulate API encounters
+    invalid processors. This change improves error handling and provides clearer feedback
+    on request issues.
+  notable: false

docs/changelog/132853.yaml

@@ -0,0 +1,5 @@
+pr: 132853
+summary: Bump bcpkix version
+area: Security
+type: upgrade
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md

@@ -31,9 +31,16 @@ Combining `query` and `retrievers` is not supported.
 `normalizer` {applies_to}`stack: ga 9.1`
 :   (Optional, String)
 
-    The normalizer to use when using the [multi-field query format](../retrievers.md#multi-field-query-format).
+    The top-level normalizer to use when combining results.
     See [normalizers](#linear-retriever-normalizers) for supported values.
     Required when `query` is specified.
+    
+    When used with the [multi-field query format](../retrievers.md#multi-field-query-format) (`query` parameter), normalizes scores per [field grouping](../retrievers.md#multi-field-field-grouping).
+    Otherwise serves as the default normalizer for any sub-retriever that doesn't specify its own normalizer. Per-retriever normalizers always take precedence over the top-level normalizer.
+
+    :::{note}
+    **Top-level normalizer support for sub-retrievers**: The ability to use a top-level normalizer as a default for sub-retrievers was introduced in Elasticsearch 9.2+. In earlier versions, only per-retriever normalizers are supported.
+    :::
 
     ::::{warning}
     Avoid using `none` as that will disable normalization and may bias the result set towards lexical matches.
@@ -74,9 +81,10 @@ Each entry in the `retrievers` array specifies the following parameters:
 `normalizer`
 :   (Optional, String)
 
-    Specifies how the retriever’s score will be normalized before applying the specified `weight`.
+    Specifies how the retriever's score will be normalized before applying the specified `weight`.
     See [normalizers](#linear-retriever-normalizers) for supported values.
-    Defaults to `none`.
+    If not specified, uses the top-level `normalizer` or defaults to `none` if no top-level normalizer is set.
+    {applies_to}`stack: ga 9.2`
 
 See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever) using a linear retriever on how to independently configure and apply normalizers to retrievers.
 
@@ -94,7 +102,7 @@ The `linear` retriever supports the following normalizers:
 
 ## Example
 
-This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking.
+This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking, with a top-level normalizer applied to all retrievers.
 
 ```console
 GET my_index/_search
@@ -105,23 +113,33 @@ GET my_index/_search
         {
           "retriever": {
             "knn": {
-              ...
+              "field": "title_vector",
+              "query_vector": [0.1, 0.2, 0.3],
+              "k": 10,
+              "num_candidates": 100
             }
           },
           "weight": 5 # KNN query weighted 5x
         },
         {
           "retriever": {
             "standard": {
-              ...
+              "query": {
+                "match": {
+                  "title": "elasticsearch"
+                }
+              }
             }
           },
           "weight": 1.5 # BM25 query weighted 1.5x
         }
-      ]
+      ],
+      "normalizer": "minmax"
     }
   }
 }
 ```
 
+In this example, the `minmax` normalizer is applied to both the kNN retriever and the standard retriever. The top-level normalizer serves as a default that can be overridden by individual sub-retrievers. When using the multi-field query format, the top-level normalizer is applied to all generated inner retrievers.
+
 See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever).

docs/reference/elasticsearch/rest-apis/searching-with-query-rules.md

@@ -16,16 +16,21 @@ $$$query-rules$$$
 * Personalized metadata about users (e.g. country, language, etc)
 * A particular topic
 * A referring site
-* etc.
+
 
 Query rules define a metadata key that will be used to match the metadata provided in the [rule retriever](/reference/elasticsearch/rest-apis/retrievers/rule-retriever.md) with the criteria specified in the rule.
 
 When a query rule matches the rule metadata according to its defined criteria, the query rule action is applied to the underlying `organic` query.
 
 For example, a query rule could be defined to match a user-entered query string of `pugs` and a country `us` and promote adoptable shelter dogs if the rule query met both criteria.
 
-Rules are defined using the [query rules API](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-query_rules) and searched using the [rule retriever](/reference/elasticsearch/rest-apis/retrievers/rule-retriever.md) or the [rule query](/reference/query-languages/query-dsl/query-dsl-rule-query.md).
+You can create and manage query rules using either:
+- [Query rules API]({{es-apis}}v9/group/endpoint-query_rules)
+- [Query Rules UI](docs-content://solutions/search/query-rules-ui.md)
 
+You can search with query rules using either:
+- [Retrievers syntax](/reference/elasticsearch/rest-apis/retrievers/rule-retriever.md)
+- [Query DSL syntax](/reference/query-languages/query-dsl/query-dsl-rule-query.md)
 
 ## Rule definition [query-rule-definition]
 
@@ -68,7 +73,7 @@ The actions to take when the rule matches a query:
 Use `ids` when searching over a single index, and `docs` when searching over multiple indices. `ids` and `docs` cannot be combined in the same query.
 
 
-## Add query rules [add-query-rules]
+## Manage query rules [manage-query-rules]
 
 You can add query rules using the [Create or update query ruleset](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-query-rules-put-ruleset) call. This adds a ruleset containing one or more query rules that will be applied to queries that match their specified criteria.
 
@@ -145,6 +150,8 @@ There is a limit of 100 rules per ruleset. This can be increased up to 1000 usin
 
 You can use the [Get query ruleset](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-query-rules-get-ruleset) call to retrieve the ruleset you just created, the [List query rulesets](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-query-rules-list-rulesets) call to retrieve a summary of all query rulesets, and the [Delete query ruleset](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-query-rules-delete-ruleset) call to delete a query ruleset.
 
+To manage rules using the Query Rules UI, refer to [Manage query rules](https://www.elastic.co/docs/solutions/search/query-rules-ui#manage-existing-rules).
+
 
 ## Search using query rules [rule-query-search]