Code changes made

Author: mridula-s109

docs/reference/elasticsearch/rest-apis/retrievers.md

@@ -263,8 +263,21 @@ A retriever that normalizes and linearly combines the scores of other retrievers
 
     A list of the sub-retrievers' configuration, that we will take into account and whose result sets we will merge through a weighted sum. Each configuration can have a different weight and normalization depending on the specified retriever.
 
+`normalizer`
+:   (Optional, String)
+
+    Specifies a normalizer to be applied to all sub-retrievers. This provides a simple way to configure normalization for all retrievers at once.
+
+    The `normalizer` can be specified at the top level, at the per-retriever level, or both, with the following rules:
+
+    * If only the top-level `normalizer` is specified, it applies to all sub-retrievers.
+    * If both a top-level and a per-retriever `normalizer` are specified, the per-retriever normalizer must be identical to the top-level one. If they differ, the request will fail.
+    * If only per-retriever normalizers are specified, they can be different for each sub-retriever.
+    * If no normalizer is specified at any level, no normalization is applied.
 
-Each entry specifies the following parameters:
+    Available values are: `minmax`, `l2_norm`, and `none`. Defaults to `none`.
+
+Each entry in the `retrievers` array specifies the following parameters:
 
 `retriever`
 :   (Required, a `retriever` object)
@@ -279,19 +292,13 @@ Each entry specifies the following parameters:
 `normalizer`
 :   (Optional, String)
 
-    - Specifies how we will normalize the retriever’s scores, before applying the specified `weight`. Available values are: `minmax`, `l2_norm`, and `none`. Defaults to `none`.
+    Specifies how we will normalize this specific retriever’s scores, before applying the specified `weight`. If a top-level `normalizer` is also specified, this normalizer must be the same. Available values are: `minmax`, `l2_norm`, and `none`. Defaults to `none`.
 
     * `none`
     * `minmax` : A `MinMaxScoreNormalizer` that normalizes scores based on the following formula
-
-        ```
-        score = (score - min) / (max - min)
-        ```
-
+    `score = (score - min) / (max - min)`
     * `l2_norm` : An `L2ScoreNormalizer` that normalizes scores using the L2 norm of the score values.
 
-See also [this hybrid search example](docs-content://solutions/search/retrievers-examples.md#retrievers-examples-linear-retriever) using a linear retriever on how to independently configure and apply normalizers to retrievers.
-
 `rank_window_size`
 :   (Optional, integer)
 

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilder.java

@@ -83,14 +83,12 @@ public final class LinearRetrieverBuilder extends CompoundRetrieverBuilder<Linea
             List<LinearRetrieverComponent> retrieverComponents = args[0] == null ? List.of() : (List<LinearRetrieverComponent>) args[0];
             List<String> fields = (List<String>) args[1];
             String query = (String) args[2];
-            String normalizerName = (String) args[3];
-            ScoreNormalizer normalizer = normalizerName == null ? null : ScoreNormalizer.valueOf(normalizerName);
+            ScoreNormalizer normalizer = args[3] == null ? null : ScoreNormalizer.valueOf((String) args[3]);
             int rankWindowSize = args[4] == null ? RankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[4];
 
             int index = 0;
             float[] weights = new float[retrieverComponents.size()];
             ScoreNormalizer[] normalizers = new ScoreNormalizer[retrieverComponents.size()];
-            Arrays.fill(normalizers, normalizer);
             List<RetrieverSource> innerRetrievers = new ArrayList<>();
             for (LinearRetrieverComponent component : retrieverComponents) {
                 innerRetrievers.add(RetrieverSource.from(component.retriever));
@@ -121,8 +119,7 @@ private static float[] getDefaultWeight(List<RetrieverSource> innerRetrievers) {
     private static ScoreNormalizer[] getDefaultNormalizers(List<RetrieverSource> innerRetrievers) {
         int size = innerRetrievers != null ? innerRetrievers.size() : 0;
         ScoreNormalizer[] normalizers = new ScoreNormalizer[size];
-        Arrays.fill(normalizers, IdentityScoreNormalizer.INSTANCE);
-        return normalizers;
+        return new ScoreNormalizer[size];
     }
 
     public static LinearRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
@@ -160,17 +157,44 @@ public LinearRetrieverBuilder(
         // Use a mutable list for innerRetrievers so that we can use addChild
         super(innerRetrievers == null ? new ArrayList<>() : new ArrayList<>(innerRetrievers), rankWindowSize);
         if (weights.length != this.innerRetrievers.size()) {
-            throw new IllegalArgumentException("The number of weights must match the number of inner retrievers");
+            throw new IllegalArgumentException(
+                "["
+                    + NAME
+                    + "] the number of weights must be equal to the number of retrievers, but found ["
+                    + weights.length
+                    + "] weights and ["
+                    + this.innerRetrievers.size()
+                    + "] retrievers"
+            );
         }
         if (normalizers.length != this.innerRetrievers.size()) {
-            throw new IllegalArgumentException("The number of normalizers must match the number of inner retrievers");
+            throw new IllegalArgumentException(
+                "["
+                    + NAME
+                    + "] the number of normalizers must be equal to the number of retrievers, but found ["
+                    + normalizers.length
+                    + "] normalizers and ["
+                    + this.innerRetrievers.size()
+                    + "] retrievers"
+            );
         }
-
-        this.fields = fields == null ? null : List.copyOf(fields);
-        this.query = query;
-        this.normalizer = normalizer;
         this.weights = weights;
         this.normalizers = normalizers;
+        this.fields = fields;
+        this.query = query;
+        this.normalizer = normalizer;
+
+        if (normalizer != null) {
+            for (ScoreNormalizer subNormalizer : normalizers) {
+                if (subNormalizer != null && subNormalizer.equals(DEFAULT_NORMALIZER) == false && subNormalizer.equals(normalizer) == false) {
+                    throw new IllegalArgumentException(
+                        "top-level normalizer ["
+                            + normalizer.getName()
+                            + "] is specified and it should be the same as all sub-retriever normalizers"
+                    );
+                }
+            }
+        }
     }
 
     public LinearRetrieverBuilder(
@@ -228,8 +252,9 @@ public ActionRequestValidationException validate(
 
         if (normalizer != null) {
             for (ScoreNormalizer perRetrieverNormalizer : normalizers) {
-                boolean isExplicitSubNormalizer = perRetrieverNormalizer != null && !perRetrieverNormalizer.equals(DEFAULT_NORMALIZER);
-                boolean isMismatch = isExplicitSubNormalizer && !perRetrieverNormalizer.equals(normalizer);
+                boolean isExplicitSubNormalizer = perRetrieverNormalizer != null
+                    && perRetrieverNormalizer.equals(DEFAULT_NORMALIZER) == false;
+                boolean isMismatch = isExplicitSubNormalizer && perRetrieverNormalizer.equals(normalizer) == false;
                 if (isMismatch) {
                     validationException = addValidationError(
                         String.format(
@@ -383,10 +408,26 @@ protected RetrieverBuilder doRewrite(QueryRewriteContext ctx) {
         }
         if (normalizer != null) {
             ScoreNormalizer[] newNormalizers = new ScoreNormalizer[normalizers.length];
-            for (int i = 0; i < normalizers.length; i++) {
-                newNormalizers[i] = (normalizers[i] == null || normalizers[i].equals(DEFAULT_NORMALIZER)) ? normalizer : normalizers[i];
-            }
-            return new LinearRetrieverBuilder(innerRetrievers, fields, query, null, rankWindowSize, weights, newNormalizers);
+            Arrays.fill(newNormalizers, normalizer);
+            rewritten = new LinearRetrieverBuilder(
+                rewritten.innerRetrievers,
+                rewritten.fields,
+                rewritten.query,
+                null,
+                rewritten.rankWindowSize,
+                rewritten.weights,
+                newNormalizers
+            );
+        } else {
+            rewritten = new LinearRetrieverBuilder(
+                rewritten.innerRetrievers,
+                rewritten.fields,
+                rewritten.query,
+                rewritten.normalizer,
+                rewritten.rankWindowSize,
+                rewritten.weights,
+                rewritten.normalizers
+            );
         }
 
         return rewritten;
@@ -413,7 +454,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept
                 builder.startObject();
                 builder.field(LinearRetrieverComponent.RETRIEVER_FIELD.getPreferredName(), entry.retriever());
                 builder.field(LinearRetrieverComponent.WEIGHT_FIELD.getPreferredName(), weights[index]);
-                if (normalizers[index] != null && !normalizers[index].equals(DEFAULT_NORMALIZER)) {
+                if (normalizers[index] != null && normalizers[index].equals(DEFAULT_NORMALIZER) == false) {
                     builder.field(LinearRetrieverComponent.NORMALIZER_FIELD.getPreferredName(), normalizers[index].getName());
                 }
                 builder.endObject();
@@ -432,7 +473,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept
         if (query != null) {
             builder.field(QUERY_FIELD.getPreferredName(), query);
         }
-        if (normalizer != null && !normalizer.equals(DEFAULT_NORMALIZER)) {
+        if (normalizer != null && normalizer.equals(DEFAULT_NORMALIZER) == false) {
             builder.field(NORMALIZER_FIELD.getPreferredName(), normalizer.getName());
         }
 

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverComponent.java

@@ -38,7 +38,7 @@ public LinearRetrieverComponent(RetrieverBuilder retrieverBuilder, Float weight,
         assert retrieverBuilder != null;
         this.retriever = retrieverBuilder;
         this.weight = weight == null ? DEFAULT_WEIGHT : weight;
-        this.normalizer = normalizer == null ? DEFAULT_NORMALIZER : normalizer;
+        this.normalizer = normalizer;
         if (this.weight < 0) {
             throw new IllegalArgumentException("[weight] must be non-negative");
         }
@@ -48,7 +48,7 @@ public LinearRetrieverComponent(RetrieverBuilder retrieverBuilder, Float weight,
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(RETRIEVER_FIELD.getPreferredName(), retriever);
         builder.field(WEIGHT_FIELD.getPreferredName(), weight);
-        if (normalizer != null && !normalizer.equals(DEFAULT_NORMALIZER)) {
+        if (normalizer != null && normalizer.equals(DEFAULT_NORMALIZER) == false) {
             builder.field(NORMALIZER_FIELD.getPreferredName(), normalizer.getName());
         }
         return builder;

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/linear/10_linear_retriever.yml

@@ -1333,3 +1333,120 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "1" }
   - close_to: { hits.hits.0._score: { value: 1.0, error: 0.001} }
+
+---
+"top level normalizer should apply to all sub-retrievers":
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            linear:
+              normalizer: "minmax"
+              retrievers: [
+                {
+                  retriever: {
+                    standard: {
+                      query: {
+                        bool: {
+                          should: [
+                            { constant_score: { filter: { term: { keyword: { value: "one" } } }, boost: 10.0 } },
+                            { constant_score: { filter: { term: { keyword: { value: "two" } } }, boost: 9.0 } },
+                            { constant_score: { filter: { term: { keyword: { value: "three" } } }, boost: 5.0 } }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  weight: 10.0
+                },
+                {
+                  retriever: {
+                    knn: {
+                      field: "vector",
+                      query_vector: [ 4 ],
+                      k: 1,
+                      num_candidates: 1
+                    }
+                  },
+                  weight: 2.0
+                }
+              ]
+
+  - match: { hits.total.value: 4 }
+  - match: { hits.hits.0._id: "1" }
+  - close_to: { hits.hits.0._score: { value: 10.0, error: 0.001} }
+  - match: { hits.hits.1._id: "2" }
+  - close_to: { hits.hits.1._score: { value: 8.0, error: 0.001} }
+  - match: { hits.hits.2._id: "4" }
+  - close_to: { hits.hits.2._score: { value: 2.0, error: 0.001} }
+  - match: { hits.hits.3._id: "3" }
+  - close_to: { hits.hits.3._score: { value: 0.0, error: 0.001 } }
+
+---
+"should throw on different normalizers":
+  - do:
+      catch: /top-level normalizer \[minmax\] is specified and it should be the same as all sub-retriever normalizers/
+      search:
+        index: test
+        body:
+          retriever:
+            linear:
+              normalizer: "minmax"
+              retrievers: [
+                {
+                  retriever: {
+                    standard: {
+                      query: {
+                        constant_score: {
+                          filter: {
+                            term: {
+                              keyword: {
+                                value: "one"
+                              }
+                            }
+                          },
+                          boost: 10.0
+                        }
+                      }
+                    }
+                  },
+                  weight: 1.0,
+                  normalizer: "l2_norm"
+                }
+              ]
+
+---
+"should not throw on same normalizers":
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            linear:
+              normalizer: "minmax"
+              retrievers: [
+                {
+                  retriever: {
+                    standard: {
+                      query: {
+                        constant_score: {
+                          filter: {
+                            term: {
+                              keyword: {
+                                value: "one"
+                              }
+                            }
+                          },
+                          boost: 10.0
+                        }
+                      }
+                    }
+                  },
+                  weight: 1.0,
+                  normalizer: "minmax"
+                }
+              ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "1" }
+  - close_to: { hits.hits.0._score: { value: 1.0, error: 0.001} }