Linear retriever top level option for normalizer

docs/changelog/129693.yaml

@@ -0,0 +1,5 @@
+pr: 129693
+summary: Add top level normalizer for linear retriever
+area: Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md

@@ -35,6 +35,8 @@ Combining `query` and `retrievers` is not supported.
     See [normalizers](#linear-retriever-normalizers) for supported values.
     Required when `query` is specified.
 
+    When used with `retrievers`, the top-level `normalizer` serves as a default for any sub-retriever that doesn't specify its own normalizer. Per-retriever normalizers always take precedence over the top-level normalizer when both are specified.
+
     ::::{warning}
     Avoid using `none` as that will disable normalization and may bias the result set towards lexical matches.
     See [field grouping](../retrievers.md#multi-field-field-grouping) for more information.
@@ -124,4 +126,46 @@ GET my_index/_search
 }
 ```
 
+### Using top-level normalizer
+
+This example shows how to use a top-level normalizer that applies to all sub-retrievers:
+
+```console
+GET my_index/_search
+{
+  "retriever": {
+    "linear": {
+      "retrievers": [
+        {
+          "retriever": {
+            "standard": {
+              "query": {
+                "match": {
+                  "title": "elasticsearch"
+                }
+              }
+            }
+          },
+          "weight": 1.0
+        },
+        {
+          "retriever": {
+            "knn": {
+              "field": "title_vector",
+              "query_vector": [0.1, 0.2, 0.3],
+              "k": 10,
+              "num_candidates": 100
+            }
+          },
+          "weight": 2.0
+        }
+      ],
+      "normalizer": "minmax"
+    }
+  }
+}
+```
+
+In this example, the `minmax` normalizer is applied to both the standard retriever and the kNN retriever. The top-level normalizer serves as a default that can be overridden by individual sub-retrievers. When using the multi-field query format, the top-level normalizer is applied to all generated inner retrievers.
+
 See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever).

x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java

@@ -835,4 +835,43 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         );
         assertThat(numAsyncCalls.get(), equalTo(4));
     }
+
+    public void testMixedNormalizerInheritance() throws IOException {
+        client().prepareIndex(INDEX)
+            .setId("1")
+            .setSource("field1", "elasticsearch search", "field2", "database technology", "score", 10)
+            .get();
+        client().prepareIndex(INDEX).setId("2").setSource("field1", "lucene engine", "field2", "search technology", "score", 5).get();
+        client().prepareIndex(INDEX)
+            .setId("3")
+            .setSource("field1", "information retrieval", "field2", "database search", "score", 15)
+            .get();
+        refresh(INDEX);
+
+        LinearRetrieverBuilder linearRetriever = new LinearRetrieverBuilder(
+            List.of(
+                CompoundRetrieverBuilder.RetrieverSource.from(
+                    new StandardRetrieverBuilder(QueryBuilders.matchQuery("field1", "elasticsearch"))
+                ),
+                CompoundRetrieverBuilder.RetrieverSource.from(
+                    new StandardRetrieverBuilder(QueryBuilders.matchQuery("field2", "technology"))
+                ),
+                CompoundRetrieverBuilder.RetrieverSource.from(new StandardRetrieverBuilder(QueryBuilders.matchQuery("field1", "search")))
+            ),
+            null,
+            null,
+            MinMaxScoreNormalizer.INSTANCE,
+            10,
+            new float[] { 1.0f, 1.0f, 1.0f },
+            new ScoreNormalizer[] { null, L2ScoreNormalizer.INSTANCE, null }
+        );
+
+        assertThat(linearRetriever.getNormalizers()[0], equalTo(MinMaxScoreNormalizer.INSTANCE));
+        assertThat(linearRetriever.getNormalizers()[1], equalTo(L2ScoreNormalizer.INSTANCE));
+        assertThat(linearRetriever.getNormalizers()[2], equalTo(MinMaxScoreNormalizer.INSTANCE));
+
+        assertResponse(client().prepareSearch(INDEX).setSource(new SearchSourceBuilder().retriever(linearRetriever)), searchResponse -> {
+            assertThat(searchResponse.getHits().getTotalHits().value() > 0L, is(true));
+        });
+    }
 }

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilder.java

@@ -43,6 +43,7 @@
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.rank.RankRRFFeatures.LINEAR_RETRIEVER_SUPPORTED;
+import static org.elasticsearch.xpack.rank.linear.LinearRetrieverComponent.DEFAULT_NORMALIZER;
 import static org.elasticsearch.xpack.rank.linear.LinearRetrieverComponent.DEFAULT_WEIGHT;
 
 /**
@@ -74,6 +75,16 @@ public final class LinearRetrieverBuilder extends CompoundRetrieverBuilder<Linea
     private final String query;
     private final ScoreNormalizer normalizer;
 
+    private static ScoreNormalizer resolveNormalizer(ScoreNormalizer componentNormalizer, ScoreNormalizer topLevelNormalizer) {
+        if (componentNormalizer != null) {
+            return componentNormalizer;
+        }
+        if (topLevelNormalizer != null) {
+            return topLevelNormalizer;
+        }
+        return DEFAULT_NORMALIZER;
+    }
+
     @SuppressWarnings("unchecked")
     static final ConstructingObjectParser<LinearRetrieverBuilder, RetrieverParserContext> PARSER = new ConstructingObjectParser<>(
         NAME,
@@ -92,7 +103,7 @@ public final class LinearRetrieverBuilder extends CompoundRetrieverBuilder<Linea
             for (LinearRetrieverComponent component : retrieverComponents) {
                 innerRetrievers.add(RetrieverSource.from(component.retriever));
                 weights[index] = component.weight;
-                normalizers[index] = component.normalizer;
+                normalizers[index] = resolveNormalizer(component.normalizer, normalizer);
                 index++;
             }
             return new LinearRetrieverBuilder(innerRetrievers, fields, query, normalizer, rankWindowSize, weights, normalizers);
@@ -118,7 +129,7 @@ private static float[] getDefaultWeight(List<RetrieverSource> innerRetrievers) {
     private static ScoreNormalizer[] getDefaultNormalizers(List<RetrieverSource> innerRetrievers) {
         int size = innerRetrievers != null ? innerRetrievers.size() : 0;
         ScoreNormalizer[] normalizers = new ScoreNormalizer[size];
-        Arrays.fill(normalizers, IdentityScoreNormalizer.INSTANCE);
+        Arrays.fill(normalizers, DEFAULT_NORMALIZER);
         return normalizers;
     }
 
@@ -167,7 +178,12 @@ public LinearRetrieverBuilder(
         this.query = query;
         this.normalizer = normalizer;
         this.weights = weights;
-        this.normalizers = normalizers;
+
+        // Initialize normalizers array
+        this.normalizers = new ScoreNormalizer[normalizers.length];
+        for (int i = 0; i < normalizers.length; i++) {
+            this.normalizers[i] = resolveNormalizer(normalizers[i], normalizer);
+        }
     }
 
     public LinearRetrieverBuilder(
@@ -221,19 +237,7 @@ public ActionRequestValidationException validate(
                 ),
                 validationException
             );
-        } else if (innerRetrievers.isEmpty() == false && normalizer != null) {
-            validationException = addValidationError(
-                String.format(
-                    Locale.ROOT,
-                    "[%s] [%s] cannot be provided when [%s] is specified",
-                    getName(),
-                    NORMALIZER_FIELD.getPreferredName(),
-                    RETRIEVERS_FIELD.getPreferredName()
-                ),
-                validationException
-            );
         }
-
         return validationException;
     }
 
@@ -339,7 +343,7 @@ protected RetrieverBuilder doRewrite(QueryRewriteContext ctx) {
                     for (var weightedRetriever : r) {
                         retrievers.add(weightedRetriever.retrieverSource());
                         weights[index] = weightedRetriever.weight();
-                        normalizers[index] = normalizer;
+                        normalizers[index] = resolveNormalizer(null, normalizer);
                         index++;
                     }
 
@@ -357,7 +361,7 @@ protected RetrieverBuilder doRewrite(QueryRewriteContext ctx) {
                 Arrays.fill(weights, DEFAULT_WEIGHT);
 
                 ScoreNormalizer[] normalizers = new ScoreNormalizer[fieldsInnerRetrievers.size()];
-                Arrays.fill(normalizers, normalizer);
+                Arrays.fill(normalizers, resolveNormalizer(null, normalizer));
 
                 // TODO: This is a incomplete solution as it does not address other incomplete copy issues
                 // (such as dropping the retriever name and min score)

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverComponent.java

@@ -38,7 +38,7 @@ public LinearRetrieverComponent(RetrieverBuilder retrieverBuilder, Float weight,
         assert retrieverBuilder != null;
         this.retriever = retrieverBuilder;
         this.weight = weight == null ? DEFAULT_WEIGHT : weight;
-        this.normalizer = normalizer == null ? DEFAULT_NORMALIZER : normalizer;
+        this.normalizer = normalizer; // Don't default to identity, allow null for top-level fallback
         if (this.weight < 0) {
             throw new IllegalArgumentException("[weight] must be non-negative");
         }

x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilderParsingTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -27,6 +28,7 @@
 import java.util.List;
 
 import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class LinearRetrieverBuilderParsingTests extends AbstractXContentTestCase<LinearRetrieverBuilder> {
     private static List<NamedXContentRegistry.Entry> xContentRegistryEntries;
@@ -67,7 +69,11 @@ protected LinearRetrieverBuilder createTestInstance() {
                 new CompoundRetrieverBuilder.RetrieverSource(TestRetrieverBuilder.createRandomTestRetrieverBuilder(), null)
             );
             weights[i] = randomFloat();
-            normalizers[i] = randomScoreNormalizer();
+            if (randomBoolean()) {
+                normalizers[i] = randomScoreNormalizer();
+            } else {
+                normalizers[i] = null;
+            }
         }
 
         return new LinearRetrieverBuilder(innerRetrievers, fields, query, normalizer, rankWindowSize, weights, normalizers);
@@ -108,10 +114,51 @@ protected NamedXContentRegistry xContentRegistry() {
     }
 
     private static ScoreNormalizer randomScoreNormalizer() {
-        if (randomBoolean()) {
-            return MinMaxScoreNormalizer.INSTANCE;
-        } else {
-            return IdentityScoreNormalizer.INSTANCE;
+        int random = randomInt(2);
+        return switch (random) {
+            case 0 -> MinMaxScoreNormalizer.INSTANCE;
+            case 1 -> L2ScoreNormalizer.INSTANCE;
+            default -> IdentityScoreNormalizer.INSTANCE;
+        };
+    }
+
+    public void testTopLevelNormalizer() throws IOException {
+        String json = """
+            {
+              "linear": {
+                "retrievers": [
+                  {
+                    "retriever": {
+                      "test": {
+                        "value": "test1"
+                      }
+                    },
+                    "weight": 1.0,
+                    "normalizer": "none"
+                  },
+                  {
+                    "retriever": {
+                      "test": {
+                        "value": "test2"
+                      }
+                    },
+                    "weight": 1.0,
+                    "normalizer": "none"
+                  }
+                ],
+                "normalizer": "minmax"
+              }
+            }""";
+
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(), json)) {
+            LinearRetrieverBuilder builder = doParseInstance(parser);
+            // Test that the top-level normalizer is properly applied - the individual
+            // retrievers specified "none" but should be overridden by top-level "minmax"
+            ScoreNormalizer[] normalizers = builder.getNormalizers();
+            assertEquals(2, normalizers.length);
+            for (ScoreNormalizer normalizer : normalizers) {
+                assertThat(normalizer, instanceOf(IdentityScoreNormalizer.class));
+            }
         }
     }
 }