Linear retriever top level option for normalizer

docs/changelog/129693.yaml

@@ -0,0 +1,5 @@
+pr: 129693
+summary: Add top level normalizer for linear retriever
+area: Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md

@@ -31,9 +31,12 @@ Combining `query` and `retrievers` is not supported.
 `normalizer` {applies_to}`stack: ga 9.1`
 :   (Optional, String)
 
-    The normalizer to use when using the [multi-field query format](../retrievers.md#multi-field-query-format).
+    The top-level normalizer to use when combining results.
     See [normalizers](#linear-retriever-normalizers) for supported values.
     Required when `query` is specified.
+
+    When used with the [multi-field query format](../retrievers.md#multi-field-query-format) (`query` parameter), normalizes scores per [field grouping](../retrievers.md#multi-field-field-grouping).
+    Otherwise serves as the default normalizer for any sub-retriever that doesn't specify its own normalizer. Per-retriever normalizers always take precedence over the top-level normalizer.
 
     ::::{warning}
     Avoid using `none` as that will disable normalization and may bias the result set towards lexical matches.
@@ -74,9 +77,9 @@ Each entry in the `retrievers` array specifies the following parameters:
 `normalizer`
 :   (Optional, String)
 
-    Specifies how the retriever’s score will be normalized before applying the specified `weight`.
+    Specifies how the retriever's score will be normalized before applying the specified `weight`.
     See [normalizers](#linear-retriever-normalizers) for supported values.
-    Defaults to `none`.
+    If not specified, uses the top-level `normalizer` or defaults to `none` if no top-level normalizer is set.
 
 See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever) using a linear retriever on how to independently configure and apply normalizers to retrievers.
 
@@ -94,7 +97,7 @@ The `linear` retriever supports the following normalizers:
 
 ## Example
 
-This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking.
+This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking, with a top-level normalizer applied to all retrievers.
 
 ```console
 GET my_index/_search
@@ -105,23 +108,33 @@ GET my_index/_search
         {
           "retriever": {
             "knn": {
-              ...
+              "field": "title_vector",
+              "query_vector": [0.1, 0.2, 0.3],
+              "k": 10,
+              "num_candidates": 100
             }
           },
           "weight": 5 # KNN query weighted 5x
         },
         {
           "retriever": {
             "standard": {
-              ...
+              "query": {
+                "match": {
+                  "title": "elasticsearch"
+                }
+              }
             }
           },
           "weight": 1.5 # BM25 query weighted 1.5x
         }
-      ]
+      ],
+      "normalizer": "minmax"
     }
   }
 }
 ```
 
+In this example, the `minmax` normalizer is applied to both the kNN retriever and the standard retriever. The top-level normalizer serves as a default that can be overridden by individual sub-retrievers. When using the multi-field query format, the top-level normalizer is applied to all generated inner retrievers.
+
 See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever).

x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java

@@ -835,4 +835,37 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         );
         assertThat(numAsyncCalls.get(), equalTo(4));
     }
+
+    public void testMixedNormalizerInheritance() throws IOException {
+        client().prepareIndex(INDEX).setId("1").setSource("field1", "elasticsearch only", "field2", "no technology here").get();
+        client().prepareIndex(INDEX).setId("2").setSource("field1", "no elasticsearch", "field2", "technology only").get();
+        client().prepareIndex(INDEX).setId("3").setSource("field1", "search term", "field2", "no technology").get();
+        refresh(INDEX);
+
+        LinearRetrieverBuilder linearRetriever = new LinearRetrieverBuilder(
+            List.of(
+                CompoundRetrieverBuilder.RetrieverSource.from(
+                    new StandardRetrieverBuilder(QueryBuilders.matchQuery("field1", "elasticsearch"))
+                ),
+                CompoundRetrieverBuilder.RetrieverSource.from(
+                    new StandardRetrieverBuilder(QueryBuilders.matchQuery("field2", "technology"))
+                ),
+                CompoundRetrieverBuilder.RetrieverSource.from(new StandardRetrieverBuilder(QueryBuilders.matchQuery("field1", "search")))
+            ),
+            null,
+            null,
+            MinMaxScoreNormalizer.INSTANCE,
+            10,
+            new float[] { 1.0f, 1.0f, 1.0f },
+            new ScoreNormalizer[] { null, L2ScoreNormalizer.INSTANCE, null }
+        );
+
+        assertThat(linearRetriever.getNormalizers()[0], equalTo(MinMaxScoreNormalizer.INSTANCE));
+        assertThat(linearRetriever.getNormalizers()[1], equalTo(L2ScoreNormalizer.INSTANCE));
+        assertThat(linearRetriever.getNormalizers()[2], equalTo(MinMaxScoreNormalizer.INSTANCE));
+
+        assertResponse(client().prepareSearch(INDEX).setSource(new SearchSourceBuilder().retriever(linearRetriever)), searchResponse -> {
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
+        });
+    }
 }

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/RankRRFFeatures.java

@@ -22,6 +22,7 @@
 public class RankRRFFeatures implements FeatureSpecification {
 
     public static final NodeFeature LINEAR_RETRIEVER_SUPPORTED = new NodeFeature("linear_retriever_supported");
+    public static final NodeFeature LINEAR_RETRIEVER_TOP_LEVEL_NORMALIZER = new NodeFeature("linear_retriever.top_level_normalizer");
 
     @Override
     public Set<NodeFeature> getFeatures() {
@@ -37,7 +38,8 @@ public Set<NodeFeature> getTestFeatures() {
             LINEAR_RETRIEVER_MINSCORE_FIX,
             LinearRetrieverBuilder.MULTI_FIELDS_QUERY_FORMAT_SUPPORT,
             RRFRetrieverBuilder.MULTI_FIELDS_QUERY_FORMAT_SUPPORT,
-            RRFRetrieverBuilder.WEIGHTED_SUPPORT
+            RRFRetrieverBuilder.WEIGHTED_SUPPORT,
+            LINEAR_RETRIEVER_TOP_LEVEL_NORMALIZER
         );
     }
 }

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilder.java

@@ -43,6 +43,7 @@
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.rank.RankRRFFeatures.LINEAR_RETRIEVER_SUPPORTED;
+import static org.elasticsearch.xpack.rank.linear.LinearRetrieverComponent.DEFAULT_NORMALIZER;
 import static org.elasticsearch.xpack.rank.linear.LinearRetrieverComponent.DEFAULT_WEIGHT;
 
 /**
@@ -92,7 +93,7 @@ public final class LinearRetrieverBuilder extends CompoundRetrieverBuilder<Linea
             for (LinearRetrieverComponent component : retrieverComponents) {
                 innerRetrievers.add(RetrieverSource.from(component.retriever));
                 weights[index] = component.weight;
-                normalizers[index] = component.normalizer;
+                normalizers[index] = resolveNormalizer(component.normalizer, normalizer);
                 index++;
             }
             return new LinearRetrieverBuilder(innerRetrievers, fields, query, normalizer, rankWindowSize, weights, normalizers);
@@ -118,10 +119,20 @@ private static float[] getDefaultWeight(List<RetrieverSource> innerRetrievers) {
     private static ScoreNormalizer[] getDefaultNormalizers(List<RetrieverSource> innerRetrievers) {
         int size = innerRetrievers != null ? innerRetrievers.size() : 0;
         ScoreNormalizer[] normalizers = new ScoreNormalizer[size];
-        Arrays.fill(normalizers, IdentityScoreNormalizer.INSTANCE);
+        Arrays.fill(normalizers, DEFAULT_NORMALIZER);
         return normalizers;
     }
 
+    private static ScoreNormalizer resolveNormalizer(ScoreNormalizer componentNormalizer, ScoreNormalizer topLevelNormalizer) {
+        if (componentNormalizer != null) {
+            return componentNormalizer;
+        }
+        if (topLevelNormalizer != null) {
+            return topLevelNormalizer;
+        }
+        return DEFAULT_NORMALIZER;
+    }
+
     public static LinearRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
         if (context.clusterSupportsFeature(LINEAR_RETRIEVER_SUPPORTED) == false) {
             throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]");
@@ -167,7 +178,10 @@ public LinearRetrieverBuilder(
         this.query = query;
         this.normalizer = normalizer;
         this.weights = weights;
-        this.normalizers = normalizers;
+        this.normalizers = new ScoreNormalizer[normalizers.length];
+        for (int i = 0; i < normalizers.length; i++) {
+            this.normalizers[i] = resolveNormalizer(normalizers[i], normalizer);
+        }
     }
 
     public LinearRetrieverBuilder(
@@ -221,19 +235,7 @@ public ActionRequestValidationException validate(
                 ),
                 validationException
             );
-        } else if (innerRetrievers.isEmpty() == false && normalizer != null) {
-            validationException = addValidationError(
-                String.format(
-                    Locale.ROOT,
-                    "[%s] [%s] cannot be provided when [%s] is specified",
-                    getName(),
-                    NORMALIZER_FIELD.getPreferredName(),
-                    RETRIEVERS_FIELD.getPreferredName()
-                ),
-                validationException
-            );
         }
-
         return validationException;
     }
 

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverComponent.java

@@ -38,7 +38,7 @@ public LinearRetrieverComponent(RetrieverBuilder retrieverBuilder, Float weight,
         assert retrieverBuilder != null;
         this.retriever = retrieverBuilder;
         this.weight = weight == null ? DEFAULT_WEIGHT : weight;
-        this.normalizer = normalizer == null ? DEFAULT_NORMALIZER : normalizer;
+        this.normalizer = normalizer; // Don't default to identity, allow null for top-level fallback
         if (this.weight < 0) {
             throw new IllegalArgumentException("[weight] must be non-negative");
         }

x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilderParsingTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -27,10 +28,17 @@
 import java.util.List;
 
 import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class LinearRetrieverBuilderParsingTests extends AbstractXContentTestCase<LinearRetrieverBuilder> {
     private static List<NamedXContentRegistry.Entry> xContentRegistryEntries;
 
+    private static final ScoreNormalizer[] SCORE_NORMALIZERS = new ScoreNormalizer[] {
+        null,
+        MinMaxScoreNormalizer.INSTANCE,
+        L2ScoreNormalizer.INSTANCE,
+        IdentityScoreNormalizer.INSTANCE };
+
     @BeforeClass
     public static void init() {
         xContentRegistryEntries = new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents();
@@ -67,7 +75,7 @@ protected LinearRetrieverBuilder createTestInstance() {
                 new CompoundRetrieverBuilder.RetrieverSource(TestRetrieverBuilder.createRandomTestRetrieverBuilder(), null)
             );
             weights[i] = randomFloat();
-            normalizers[i] = randomScoreNormalizer();
+            normalizers[i] = randomFrom(SCORE_NORMALIZERS);
         }
 
         return new LinearRetrieverBuilder(innerRetrievers, fields, query, normalizer, rankWindowSize, weights, normalizers);
@@ -108,10 +116,46 @@ protected NamedXContentRegistry xContentRegistry() {
     }
 
     private static ScoreNormalizer randomScoreNormalizer() {
-        if (randomBoolean()) {
-            return MinMaxScoreNormalizer.INSTANCE;
-        } else {
-            return IdentityScoreNormalizer.INSTANCE;
+        return randomFrom(SCORE_NORMALIZERS);
+    }
+
+    public void testTopLevelNormalizer() throws IOException {
+        String json = """
+            {
+              "linear": {
+                "retrievers": [
+                  {
+                    "retriever": {
+                      "test": {
+                        "value": "test1"
+                      }
+                    },
+                    "weight": 1.0,
+                    "normalizer": "none"
+                  },
+                  {
+                    "retriever": {
+                      "test": {
+                        "value": "test2"
+                      }
+                    },
+                    "weight": 1.0,
+                    "normalizer": "none"
+                  }
+                ],
+                "normalizer": "minmax"
+              }
+            }""";
+
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(), json)) {
+            LinearRetrieverBuilder builder = doParseInstance(parser);
+            // Test that the top-level normalizer is properly applied - the individual
+            // Per-retriever 'none' should override top-level 'minmax'
+            ScoreNormalizer[] normalizers = builder.getNormalizers();
+            assertEquals(2, normalizers.length);
+            for (ScoreNormalizer normalizer : normalizers) {
+                assertThat(normalizer, instanceOf(IdentityScoreNormalizer.class));
+            }
         }
     }
 }