Merge branch 'main' into markjhoy/default_token_pruning_sparse_vector

markjhoy · web-flow · commit 99c3700d1061 · 2025-05-01T09:08:24.000-04:00
diff --git a/docs/changelog/126401.yaml b/docs/changelog/126401.yaml
@@ -0,0 +1,5 @@
+pr: 126401
+summary: Add pinned retriever
+area: Relevance
+type: enhancement
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -231,7 +231,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00);
     public static final TransportVersion PROJECT_METADATA_SETTINGS = def(9_066_00_0);
     public static final TransportVersion AGGREGATE_METRIC_DOUBLE_BLOCK = def(9_067_00_0);
-    public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS = def(9_068_0_00);
+    public static final TransportVersion PINNED_RETRIEVER = def(9_068_0_00);
+    public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS = def(9_069_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/ent-search/build.gradle b/x-pack/plugin/ent-search/build.gradle
@@ -8,7 +8,7 @@ esplugin {
   name = 'x-pack-ent-search'
   description = 'Elasticsearch Expanded Pack Plugin - Enterprise Search'
   classname = 'org.elasticsearch.xpack.application.EnterpriseSearch'
-  extendedPlugins = ['x-pack-core']
+  extendedPlugins = ['x-pack-core', 'search-business-rules']
 }
 
 base {
@@ -17,7 +17,7 @@ base {
 
 dependencies {
   compileOnly project(path: xpackModule('core'))
-  implementation project(xpackModule('search-business-rules'))
+  compileOnly project(xpackModule('search-business-rules'))
   api project(':modules:lang-mustache')
 
   // JSON Schema dependencies
diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
@@ -14,6 +14,12 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
 apply plugin: 'elasticsearch.bwc-test'
+apply plugin: org.elasticsearch.gradle.internal.precommit.CheckstylePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenApisPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.LoggerUsagePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin
 
 restResources {
   restApi {
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle
@@ -10,6 +10,12 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
+apply plugin: org.elasticsearch.gradle.internal.precommit.CheckstylePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenApisPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.LoggerUsagePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin
 
 dependencies {
   javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
diff --git a/x-pack/plugin/esql/qa/server/multi-node/build.gradle b/x-pack/plugin/esql/qa/server/multi-node/build.gradle
@@ -3,6 +3,12 @@ import org.elasticsearch.gradle.util.GradleUtils
 apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: org.elasticsearch.gradle.internal.precommit.CheckstylePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenApisPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.LoggerUsagePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin
 
 dependencies {
   javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle
@@ -2,6 +2,12 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 // Necessary to use tests in Serverless
 apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: org.elasticsearch.gradle.internal.precommit.CheckstylePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenApisPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.LoggerUsagePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin
 
 dependencies {
   javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java
@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.regex.Pattern;
 
@@ -164,7 +165,7 @@ private void indexValue(String value) throws IOException {
 
         Request bulk = new Request("POST", "/_bulk");
         bulk.addParameter("refresh", "");
-        bulk.setJsonEntity(String.format("""
+        bulk.setJsonEntity(String.format(Locale.ROOT, """
             {"create":{"_index":"test"}}
             {"test":"%s"}
             """, value));
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
@@ -707,13 +707,13 @@ public void testSuggestedCast() throws IOException {
                         "default_metric": "max"
                     """;
             }
-            createIndex("index-" + type.esType(), null, """
+            createIndex("index-" + type.esType(), null, String.format(Locale.ROOT, """
                  "properties": {
                    "my_field": {
                      "type": "%s" %s
                    }
                  }
-                """.formatted(type.esType(), additionalProperties));
+                """, type.esType(), additionalProperties));
             Request doc = new Request("PUT", "index-" + type.esType() + "/_doc/1");
             doc.setJsonEntity("{\"my_field\": " + typesAndValues.get(type) + "}");
             client().performRequest(doc);
@@ -724,11 +724,11 @@ public void testSuggestedCast() throws IOException {
 
         for (int i = 0; i < listOfTypes.size(); i++) {
             for (int j = i + 1; j < listOfTypes.size(); j++) {
-                String query = """
+                String query = String.format(Locale.ROOT, """
                     {
                         "query": "FROM index-%s,index-%s | LIMIT 100 | KEEP my_field"
                     }
-                    """.formatted(listOfTypes.get(i).esType(), listOfTypes.get(j).esType());
+                    """, listOfTypes.get(i).esType(), listOfTypes.get(j).esType());
                 Request request = new Request("POST", "/_query");
                 request.setJsonEntity(query);
                 Response resp = client().performRequest(request);
@@ -749,11 +749,13 @@ public void testSuggestedCast() throws IOException {
                     )
                 );
 
-                String castedQuery = """
-                    {
-                        "query": "FROM index-%s,index-%s | LIMIT 100 | EVAL my_field = my_field::%s"
-                    }
-                    """.formatted(
+                String castedQuery = String.format(
+                    Locale.ROOT,
+                    """
+                        {
+                            "query": "FROM index-%s,index-%s | LIMIT 100 | EVAL my_field = my_field::%s"
+                        }
+                        """,
                     listOfTypes.get(i).esType(),
                     listOfTypes.get(j).esType(),
                     suggestedCast == DataType.KEYWORD ? "STRING" : suggestedCast.nameUpper()
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/StoredFieldsSequentialIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/StoredFieldsSequentialIT.java
@@ -38,7 +38,8 @@
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.runEsql;
 import static org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT.commonProfile;
 import static org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT.fixTypesOnProfile;
-import static org.hamcrest.Matchers.*;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 /**
  * Tests for {@code index.esql.stored_fields_sequential_proportion} which controls
diff --git a/x-pack/plugin/rank-rrf/build.gradle b/x-pack/plugin/rank-rrf/build.gradle
@@ -26,6 +26,7 @@ dependencies {
   clusterModules project(xpackModule('rank-rrf'))
   clusterModules project(xpackModule('inference'))
   clusterModules project(':modules:lang-painless')
+  clusterModules project(xpackModule('search-business-rules'))
 
   clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
 }
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml
@@ -0,0 +1,98 @@
+setup:
+  - requires:
+      cluster_features: 'pinned_retriever_supported'
+      reason: 'test requires pinned retriever implementation'
+  - do:
+      indices.create:
+        index: test-index1
+
+  - do:
+      bulk:
+        refresh: true
+        index: test-index1
+        body:
+          - index:
+              _id: doc1
+          - { "text": "document one" }
+          - index:
+              _id: doc2
+          - { "text": "document two" }
+          - index:
+              _id: doc3
+          - { "text": "document three" }
+          - index:
+              _id: doc4
+          - { "text": "document four" }
+          - index:
+              _id: doc5
+          - { "text": "document five" }
+
+---
+"rrf combined with pinned retriever":
+  - skip: { features: headers }
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        index: test-index1
+        body:
+          retriever:
+            pinned:
+              ids: ["doc1"]
+              retriever:
+                rrf:
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          term: { text: "document" }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        query: {
+                          term: { text: "three" }
+                        }
+                      }
+                    }
+                  ]
+              rank_window_size: 10
+
+  - match: { hits.total.value: 5 }
+  - match: { hits.hits.0._id: doc1 }
+  - match: { hits.hits.0._score: 1.7014122E38 }
+  - match: { hits.hits.1._id: doc3 }
+  - match: { hits.hits.1._score < 100.0 }
+  - match: { hits.hits.2._id: doc2 }
+
+---
+"rrf with pinned retriever as a sub-retriever":
+  - skip: { features: headers }
+  - do:
+      headers: { Content-Type: application/json }
+      search:
+        index: test-index1
+        body:
+          retriever:
+            rrf:
+              retrievers:
+                -
+                  standard:
+                    query:
+                      match: { text: "document" }
+                -
+                  pinned:
+                    ids: ["doc4", "doc5"]
+                    retriever:
+                      standard:
+                        query:
+                          match: { text: "document" }
+
+  - match: { hits.total.value: 5 }
+  - match: { hits.hits.0._id: doc1 }
+  - lt: { hits.hits.0._score: 100.0 }
+  - match: { hits.hits.1._id: doc4 }
+  - match: { hits.hits.2._id: doc5 }
+
+
diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle
@@ -1,5 +1,6 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
   name = 'search-business-rules'
@@ -13,6 +14,19 @@ base {
 
 dependencies {
   compileOnly project(path: xpackModule('core'))
+  compileOnly project(':server')
   testImplementation(testArtifact(project(xpackModule('core'))))
-  testImplementation project(":test:framework")
+  testImplementation(testArtifact(project(':server')))
+  clusterModules project(xpackModule('search-business-rules'))
+  clusterModules project(':modules:mapper-extras')
+  clusterModules project(':modules:lang-painless')
+  clusterModules project(xpackModule('inference'))
 }
+
+tasks.named("yamlRestTest") {
+  usesDefaultDistribution("uses search business rules plugin")
+}
+artifacts {
+  restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
+}
+
diff --git a/x-pack/plugin/search-business-rules/src/main/java/module-info.java b/x-pack/plugin/search-business-rules/src/main/java/module-info.java
@@ -16,4 +16,7 @@
     requires org.elasticsearch.xcore;
 
     exports org.elasticsearch.xpack.searchbusinessrules;
+
+    provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures;
+    provides org.elasticsearch.plugins.SearchPlugin with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules;
 }
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java
@@ -59,7 +59,7 @@ public class PinnedQueryBuilder extends AbstractQueryBuilder<PinnedQueryBuilder>
 
     // Organic queries will have their scores capped to this number range,
     // We reserve the highest float exponent for scores of pinned queries
-    private static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1;
+    public static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1;
 
     public PinnedQueryBuilder(QueryBuilder organicQuery, String... ids) {
         this(organicQuery, Arrays.asList(ids), null);
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java
@@ -7,18 +7,28 @@
 
 package org.elasticsearch.xpack.searchbusinessrules;
 
+import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.plugins.SearchPlugin.QuerySpec;
+import org.elasticsearch.plugins.SearchPlugin.RetrieverSpec;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder;
 
 import java.util.List;
 
 import static java.util.Collections.singletonList;
 
-public class SearchBusinessRules extends Plugin implements SearchPlugin {
+public class SearchBusinessRules extends Plugin implements SearchPlugin, ExtensiblePlugin {
 
     @Override
     public List<QuerySpec<?>> getQueries() {
-        return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent));
+        return List.of(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent));
+    }
+
+    @Override
+    public List<RetrieverSpec<?>> getRetrievers() {
+        return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent));
     }
 
 }
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.searchbusinessrules;
+
+import org.elasticsearch.features.FeatureSpecification;
+import org.elasticsearch.features.NodeFeature;
+
+import java.util.Set;
+
+import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE;
+
+public class SearchBusinessRulesFeatures implements FeatureSpecification {
+
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of();
+    }
+
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(PINNED_RETRIEVER_FEATURE);
+    }
+}
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java
@@ -94,7 +94,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder.endObject();
     }
 
-    static final ConstructingObjectParser<SpecifiedDocument, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<SpecifiedDocument, Void> PARSER = new ConstructingObjectParser<>(
         NAME,
         a -> new SpecifiedDocument((String) a[0], (String) a[1])
     );
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java
diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java
diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java
diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml
