Merge remote-tracking branch 'origin/main' into feature/misleading-error-message

Author: ankit--sethi

docs/changelog/127563.yaml

@@ -0,0 +1,6 @@
+pr: 127563
+summary: "ESQL: Avoid unintended attribute removal"
+area: ES|QL
+type: bug
+issues:
+  - 127468

docs/changelog/128153.yaml

@@ -0,0 +1,6 @@
+pr: 128153
+summary: "Fix: Add `NamedWriteable` for `RuleQueryRankDoc`"
+area: Relevance
+type: bug
+issues:
+ - 126071

docs/reference/query-languages/esql/esql-functions-operators.md

@@ -12,56 +12,67 @@ mapped_pages:
 ## Functions overview [esql-functions]
 
 ::::{dropdown} Aggregate functions
+:open:
 :::{include} _snippets/lists/aggregation-functions.md
 :::
 ::::
 
 ::::{dropdown} Grouping functions
+:open:
 :::{include} _snippets/lists/grouping-functions.md
 :::
 ::::
 
 ::::{dropdown} Conditional functions and expressions
+:open:
 :::{include} _snippets/lists/conditional-functions-and-expressions.md
 :::
 ::::
 
 ::::{dropdown} Date and time functions
+:open:
 :::{include} _snippets/lists/date-time-functions.md
 :::
 ::::
 
 ::::{dropdown} IP functions
+:open:
 :::{include} _snippets/lists/ip-functions.md
 :::
 ::::
 
 ::::{dropdown} Math functions
+:open:
 :::{include} _snippets/lists/math-functions.md
 :::
 ::::
 
 ::::{dropdown} Search functions
+:open:
 :::{include} _snippets/lists/search-functions.md
 :::
 ::::
 
 ::::{dropdown} Spatial functions
+:open:
 :::{include} _snippets/lists/spatial-functions.md
 :::
 ::::
 
 ::::{dropdown} String functions
+:open:
 :::{include} _snippets/lists/string-functions.md
 :::
 ::::
 
 ::::{dropdown} Type conversion functions
+:open:
 :::{include} _snippets/lists/type-conversion-functions.md
 :::
 ::::
 
 ::::{dropdown} Multi value functions
+:open:
 :::{include} _snippets/lists/mv-functions.md
 :::
 ::::
@@ -70,6 +81,7 @@ mapped_pages:
 ## Operators overview [esql-operators-overview]
 
 ::::{dropdown} Operators
+:open:
 :::{include} _snippets/lists/operators.md
 :::
 ::::

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -43,14 +43,14 @@ public record BootstrapArgs(
         Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
         PathLookup pathLookup,
         Map<String, Path> sourcePaths,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(scopeResolver);
             requireNonNull(pathLookup);
             requireNonNull(sourcePaths);
-            requireNonNull(suppressFailureLogClasses);
+            requireNonNull(suppressFailureLogPackages);
         }
     }
 
@@ -78,7 +78,7 @@ public static BootstrapArgs bootstrapArgs() {
      * @param tempDir        the temp directory for Elasticsearch
      * @param logsDir        the log directory for Elasticsearch
      * @param pidFile        path to a pid file for Elasticsearch, or {@code null} if one was not specified
-     * @param suppressFailureLogClasses   classes for which we do not need or want to log Entitlements failures
+     * @param suppressFailureLogPackages   packages for which we do not need or want to log Entitlements failures
      */
     public static void bootstrap(
         Policy serverPolicyPatch,
@@ -95,7 +95,7 @@ public static void bootstrap(
         Path logsDir,
         Path tempDir,
         Path pidFile,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
@@ -119,7 +119,7 @@ public static void bootstrap(
                 settingResolver
             ),
             sourcePaths,
-            suppressFailureLogClasses
+            suppressFailureLogPackages
         );
         exportInitializationToAgent();
         loadAgent(findAgentJar());

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -90,7 +90,7 @@ private static PolicyManager createPolicyManager() {
             EntitlementBootstrap.bootstrapArgs().sourcePaths(),
             ENTITLEMENTS_MODULE,
             pathLookup,
-            bootstrapArgs.suppressFailureLogClasses()
+            bootstrapArgs.suppressFailureLogPackages()
         );
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -256,7 +256,7 @@ ModuleEntitlements policyEntitlements(String componentName, Path componentPath,
     private final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, PolicyScope> scopeResolver;
     private final PathLookup pathLookup;
-    private final Set<Class<?>> mutedClasses;
+    private final Set<Package> suppressFailureLogPackages;
 
     public static final String ALL_UNNAMED = "ALL-UNNAMED";
 
@@ -311,7 +311,7 @@ public PolicyManager(
         Map<String, Path> sourcePaths,
         Module entitlementsModule,
         PathLookup pathLookup,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy));
         this.apmAgentEntitlements = apmAgentEntitlements;
@@ -322,7 +322,7 @@ public PolicyManager(
         this.sourcePaths = sourcePaths;
         this.entitlementsModule = entitlementsModule;
         this.pathLookup = requireNonNull(pathLookup);
-        this.mutedClasses = suppressFailureLogClasses;
+        this.suppressFailureLogPackages = suppressFailureLogPackages;
 
         List<ExclusiveFileEntitlement> exclusiveFileEntitlements = new ArrayList<>();
         for (var e : serverEntitlements.entrySet()) {
@@ -688,8 +688,8 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
 
     private void notEntitled(String message, Class<?> callerClass, ModuleEntitlements entitlements) {
         var exception = new NotEntitledException(message);
-        // Don't emit a log for muted classes, e.g. classes containing self tests
-        if (mutedClasses.contains(callerClass) == false) {
+        // Don't emit a log for suppressed packages, e.g. packages containing self tests
+        if (suppressFailureLogPackages.contains(callerClass.getPackage()) == false) {
             entitlements.logger().warn("Not entitled: {}", message, exception);
         }
         throw exception;

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java

@@ -26,13 +26,15 @@ public class DataStreamFeatures implements FeatureSpecification {
         "data_stream.downsample.default_aggregate_metric_fix"
     );
 
+    public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream");
+
     @Override
     public Set<NodeFeature> getFeatures() {
         return Set.of(DataStream.DATA_STREAM_FAILURE_STORE_FEATURE);
     }
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
-        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX);
+        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE);
     }
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction;
 import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction;
 import org.elasticsearch.client.internal.OriginSettingClient;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -70,10 +71,12 @@
 import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction;
 import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction;
 import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction;
+import org.elasticsearch.datastreams.rest.RestGetDataStreamSettingsAction;
 import org.elasticsearch.datastreams.rest.RestGetDataStreamsAction;
 import org.elasticsearch.datastreams.rest.RestMigrateToDataStreamAction;
 import org.elasticsearch.datastreams.rest.RestModifyDataStreamsAction;
 import org.elasticsearch.datastreams.rest.RestPromoteDataStreamAction;
+import org.elasticsearch.datastreams.rest.RestUpdateDataStreamSettingsAction;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.health.HealthIndicatorService;
 import org.elasticsearch.index.IndexSettingProvider;
@@ -280,6 +283,10 @@ public List<RestHandler> getRestHandlers(
         handlers.add(new RestGetDataStreamOptionsAction());
         handlers.add(new RestPutDataStreamOptionsAction());
         handlers.add(new RestDeleteDataStreamOptionsAction());
+        if (DataStream.LOGS_STREAM_FEATURE_FLAG) {
+            handlers.add(new RestGetDataStreamSettingsAction());
+            handlers.add(new RestUpdateDataStreamSettingsAction());
+        }
         return handlers;
     }
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.datastreams.rest;
+
+import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
+import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestGetDataStreamSettingsAction extends BaseRestHandler {
+    @Override
+    public String getName() {
+        return "get_data_stream_settings_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(GET, "/_data_stream/{name}/_settings"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        GetDataStreamSettingsAction.Request getDataStreamRequest = new GetDataStreamSettingsAction.Request(
+            RestUtils.getMasterNodeTimeout(request)
+        ).indices(Strings.splitStringByCommaToArray(request.param("name")));
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
+            GetDataStreamSettingsAction.INSTANCE,
+            getDataStreamRequest,
+            new RestRefCountedChunkedToXContentListener<>(channel)
+        );
+    }
+}

modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.datastreams.rest;
+
+import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
+import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestUpdateDataStreamSettingsAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "update_data_stream_settings_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(PUT, "/_data_stream/{name}/_settings"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        Settings settings;
+        try (XContentParser parser = request.contentParser()) {
+            settings = Settings.fromXContent(parser);
+        }
+        UpdateDataStreamSettingsAction.Request putDataStreamRequest = new UpdateDataStreamSettingsAction.Request(
+            settings,
+            RestUtils.getMasterNodeTimeout(request),
+            RestUtils.getAckTimeout(request)
+        ).indices(Strings.splitStringByCommaToArray(request.param("name")));
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
+            UpdateDataStreamSettingsAction.INSTANCE,
+            putDataStreamRequest,
+            new RestRefCountedChunkedToXContentListener<>(channel)
+        );
+    }
+}