Merge branch 'main' into feature/timestamp-doc-values-spars-index-setting

Author: salvatore-campagna

muted-tests.yml

@@ -388,14 +388,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests
   method: testSnapshotRecovery {p0=false p1=true}
   issue: https://github.com/elastic/elasticsearch/issues/122551
-- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT
-  issue: https://github.com/elastic/elasticsearch/issues/122566
-- class: org.elasticsearch.entitlement.qa.EntitlementsAllowedIT
-  issue: https://github.com/elastic/elasticsearch/issues/122567
-- class: org.elasticsearch.entitlement.qa.EntitlementsAllowedNonModularIT
-  issue: https://github.com/elastic/elasticsearch/issues/122568
-- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedNonModularIT
-  issue: https://github.com/elastic/elasticsearch/issues/122569
 
 # Examples:
 #

x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java

@@ -36,6 +36,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
@@ -148,16 +149,28 @@ public void chunkedInfer(
         }
 
         private RankedDocsResults makeResults(List<String> input) {
-            List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
             int totalResults = input.size();
-            float minScore = random.nextFloat(-1f, 1f);
-            float resultDiff = 0.2f;
-            for (int i = 0; i < input.size(); i++) {
-                results.add(
-                    new RankedDocsResults.RankedDoc(totalResults - 1 - i, minScore + resultDiff * (totalResults - i), input.get(i))
-                );
+            try {
+                List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
+                for (int i = 0; i < totalResults; i++) {
+                    results.add(new RankedDocsResults.RankedDoc(i, Float.parseFloat(input.get(i)), input.get(i)));
+                }
+                return new RankedDocsResults(results.stream().sorted(Comparator.reverseOrder()).toList());
+            } catch (NumberFormatException ex) {
+                List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
+                float minScore = random.nextFloat(-1f, 1f);
+                float resultDiff = 0.2f;
+                for (int i = 0; i < input.size(); i++) {
+                    results.add(
+                        new RankedDocsResults.RankedDoc(
+                            totalResults - 1 - i,
+                            minScore + resultDiff * (totalResults - i),
+                            input.get(totalResults - 1 - i)
+                        )
+                    );
+                }
+                return new RankedDocsResults(results);
             }
-            return new RankedDocsResults(results);
         }
 
         protected ServiceSettings getServiceSettingsFromMap(Map<String, Object> serviceSettingsMap) {

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -27,6 +27,9 @@ public class InferenceFeatures implements FeatureSpecification {
 
     private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER = new NodeFeature("semantic_text.highlighter");
     private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT = new NodeFeature("semantic_text.highlighter.default");
+    private static final NodeFeature TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE = new NodeFeature(
+        "test_reranking_service.parse_text_as_score"
+    );
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -45,7 +48,8 @@ public Set<NodeFeature> getTestFeatures() {
             TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX,
             SemanticInferenceMetadataFieldsMapper.INFERENCE_METADATA_FIELDS_ENABLED_BY_DEFAULT,
             SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT,
-            SEMANTIC_KNN_FILTER_FIX
+            SEMANTIC_KNN_FILTER_FIX,
+            TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE
         );
     }
 }

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml

@@ -2,7 +2,6 @@ setup:
   - skip:
       features:
         - close_to
-        - contains
   - requires:
       test_runner_features: "close_to"
 
@@ -33,16 +32,8 @@ setup:
                 type: keyword
               subtopic:
                 type: keyword
-
-  - do:
-      index:
-        index: test-index
-        id: doc_1
-        body:
-          text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun."
-          topic: [ "science" ]
-          subtopic: [ "technology" ]
-        refresh: true
+              inference_text_field:
+                type: text
 
   - do:
       index:
@@ -52,6 +43,7 @@ setup:
           text: "The phases of the Moon come from the position of the Moon relative to the Earth and Sun."
           topic: [ "science" ]
           subtopic: [ "astronomy" ]
+          inference_text_field: "0"
         refresh: true
 
   - do:
@@ -61,11 +53,27 @@ setup:
         body:
           text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan."
           topic: [ "geography" ]
+          inference_text_field: "1"
+        refresh: true
+
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun."
+          topic: [ "science" ]
+          subtopic: [ "technology" ]
+          inference_text_field: "-1"
         refresh: true
 
 ---
 "Simple text similarity rank retriever":
 
+  - requires:
+      cluster_features: "test_reranking_service.parse_text_as_score"
+      reason: test_reranking_service can now parse provided input as score to provide deterministic ranks
+
   - do:
       search:
         index: test-index
@@ -75,14 +83,37 @@ setup:
           retriever:
             text_similarity_reranker:
               retriever:
+                # this one returns docs 1 and 2
                 standard:
                   query:
-                    term:
-                      topic: "science"
+                    bool: {
+                      should: [
+                        {
+                          constant_score: {
+                            filter: {
+                              term: {
+                                subtopic: "technology"
+                              }
+                            },
+                            boost: 10
+                          }
+                        },
+                        {
+                          constant_score: {
+                            filter: {
+                              term: {
+                                subtopic: "astronomy"
+                              }
+                            },
+                            boost: 1
+                          }
+                        }
+                      ]
+                    }
               rank_window_size: 10
               inference_id: my-rerank-model
               inference_text: "How often does the moon hide the sun?"
-              field: text
+              field: inference_text_field
           size: 10
 
   - match: { hits.total.value: 2 }
@@ -94,6 +125,10 @@ setup:
 ---
 "Simple text similarity rank retriever and filtering":
 
+  - requires:
+      cluster_features: "test_reranking_service.parse_text_as_score"
+      reason: test_reranking_service can now parse provided input as score to provide deterministic ranks
+
   - do:
       search:
         index: test-index
@@ -103,6 +138,7 @@ setup:
           retriever:
             text_similarity_reranker:
               retriever:
+                # this one returns doc 1
                 standard:
                   query:
                     term:
@@ -113,7 +149,7 @@ setup:
               rank_window_size: 10
               inference_id: my-rerank-model
               inference_text: "How often does the moon hide the sun?"
-              field: text
+              field: inference_text_field
           size: 10
 
   - match: { hits.total.value: 1 }
@@ -143,7 +179,7 @@ setup:
               rank_window_size: 10
               inference_id: i-dont-exist
               inference_text: "How often does the moon hide the sun?"
-              field: text
+              field: inference_text_field
           size: 10
 
 ---
@@ -169,13 +205,17 @@ setup:
               rank_window_size: 10
               inference_id: i-dont-exist
               inference_text: "asdfasdf"
-              field: text
+              field: inference_text_field
           size: 10
 
 
 ---
 "text similarity reranking with explain":
 
+  - requires:
+      cluster_features: "test_reranking_service.parse_text_as_score"
+      reason: test_reranking_service can now parse provided input as score to provide deterministic ranks
+
   - do:
       search:
         index: test-index
@@ -186,28 +226,50 @@ setup:
             text_similarity_reranker: {
               retriever:
                 {
+                  # this one returns doc 1 and 2
                   standard: {
                     query: {
-                      term: {
-                        topic: "science"
+                      bool: {
+                        should: [
+                          {
+                            constant_score: {
+                              filter: {
+                                term: {
+                                  subtopic: "technology"
+                                }
+                              },
+                              boost: 10
+                            }
+                          },
+                          {
+                            constant_score: {
+                              filter: {
+                                term: {
+                                  subtopic: "astronomy"
+                                }
+                              },
+                              boost: 1
+                            }
+                          }
+                        ]
                       }
                     }
                   }
                 },
               rank_window_size: 10,
               inference_id: my-rerank-model,
               inference_text: "How often does the moon hide the sun?",
-              field: text
+              field: inference_text_field
             }
           }
           size: 10
           explain: true
 
-  - contains: { hits.hits: { _id: "doc_2" } }
-  - contains: { hits.hits: { _id: "doc_1" } }
+  - match: { hits.hits.0._id: "doc_2" }
+  - match: { hits.hits.1._id: "doc_1" }
 
-  - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" }
-  - match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" }
+  - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[inference_text_field\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/subtopic.*astronomy.*/" }
 
 ---
 "text similarity reranker properly handles aliases":
@@ -281,7 +343,7 @@ setup:
               rank_window_size: 10
               inference_id: my-rerank-model
               inference_text: "How often does the moon hide the sun?"
-              field: text
+              field: inference_text_field
           size: 10
 
   - match: { hits.total.value: 1 }

x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java

@@ -235,7 +235,7 @@ public void testLogsdbRouteOnSortFields() throws IOException {
         var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
         assertEquals("logsdb", settings.get("index.mode"));
         assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
-        assertEquals("true", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
-        assertEquals(List.of("host.name", "message"), settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
+        assertEquals("false", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
+        assertNull(settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
     }
 }

x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 
 public class LogsdbRestIT extends ESRestTestCase {
@@ -66,9 +67,15 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
             logger.info("response's features: {}", features);
             assertThat(features, Matchers.not(Matchers.empty()));
-            Map<?, ?> feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get();
-            assertThat(feature.get("name"), equalTo("synthetic-source"));
-            assertThat(feature.get("license_level"), equalTo("enterprise"));
+            boolean found = false;
+            for (var feature : features) {
+                if (feature.get("family") != null) {
+                    assertThat(feature.get("name"), anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields")));
+                    assertThat(feature.get("license_level"), equalTo("enterprise"));
+                    found = true;
+                }
+            }
+            assertTrue(found);
 
             var indexResponse = (Map<?, ?>) getIndexSettings("test-index", true).get("test-index");
             logger.info("indexResponse: {}", indexResponse);

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

@@ -40,12 +40,12 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING;
+import static org.elasticsearch.xpack.logsdb.LogsdbLicenseService.FALLBACK_SETTING;
 
 public class LogsDBPlugin extends Plugin implements ActionPlugin {
 
     private final Settings settings;
-    private final SyntheticSourceLicenseService licenseService;
+    private final LogsdbLicenseService licenseService;
     private static final Setting<Boolean> LOGSDB_PRIOR_LOGS_USAGE = Setting.boolSetting(
         "logsdb.prior_logs_usage",
         false,
@@ -63,7 +63,7 @@ public class LogsDBPlugin extends Plugin implements ActionPlugin {
 
     public LogsDBPlugin(Settings settings) {
         this.settings = settings;
-        this.licenseService = new SyntheticSourceLicenseService(settings);
+        this.licenseService = new LogsdbLicenseService(settings);
         this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(licenseService, settings);
     }
 
@@ -82,7 +82,6 @@ public Collection<?> createComponents(PluginServices services) {
             CLUSTER_LOGSDB_ENABLED,
             logsdbIndexModeSettingsProvider::updateClusterIndexModeLogsdbEnabled
         );
-
         // Nothing to share here:
         return super.createComponents(services);
     }
@@ -95,6 +94,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
                 IndexVersion.current(),
                 parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion()
             ),
+            DiscoveryNode.isStateless(settings) == false,
             DiscoveryNode.isStateless(settings) == false
         );
         return List.of(logsdbIndexModeSettingsProvider);

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java

@@ -64,7 +64,7 @@ protected void masterOperation(
             }
         }
         final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());
-        final boolean hasCustomCutoffDate = System.getProperty(SyntheticSourceLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
+        final boolean hasCustomCutoffDate = System.getProperty(LogsdbLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
         final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new);
         final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes);
         final int finalNumIndices = numIndices;