Merge branch 'main' into source_provider_ref

Author: jimczi

docs/changelog/127563.yaml

@@ -0,0 +1,6 @@
+pr: 127563
+summary: "ESQL: Avoid unintended attribute removal"
+area: ES|QL
+type: bug
+issues:
+  - 127468

docs/reference/query-languages/esql/esql-functions-operators.md

@@ -12,56 +12,67 @@ mapped_pages:
 ## Functions overview [esql-functions]
 
 ::::{dropdown} Aggregate functions
+:open:
 :::{include} _snippets/lists/aggregation-functions.md
 :::
 ::::
 
 ::::{dropdown} Grouping functions
+:open:
 :::{include} _snippets/lists/grouping-functions.md
 :::
 ::::
 
 ::::{dropdown} Conditional functions and expressions
+:open:
 :::{include} _snippets/lists/conditional-functions-and-expressions.md
 :::
 ::::
 
 ::::{dropdown} Date and time functions
+:open:
 :::{include} _snippets/lists/date-time-functions.md
 :::
 ::::
 
 ::::{dropdown} IP functions
+:open:
 :::{include} _snippets/lists/ip-functions.md
 :::
 ::::
 
 ::::{dropdown} Math functions
+:open:
 :::{include} _snippets/lists/math-functions.md
 :::
 ::::
 
 ::::{dropdown} Search functions
+:open:
 :::{include} _snippets/lists/search-functions.md
 :::
 ::::
 
 ::::{dropdown} Spatial functions
+:open:
 :::{include} _snippets/lists/spatial-functions.md
 :::
 ::::
 
 ::::{dropdown} String functions
+:open:
 :::{include} _snippets/lists/string-functions.md
 :::
 ::::
 
 ::::{dropdown} Type conversion functions
+:open:
 :::{include} _snippets/lists/type-conversion-functions.md
 :::
 ::::
 
 ::::{dropdown} Multi value functions
+:open:
 :::{include} _snippets/lists/mv-functions.md
 :::
 ::::
@@ -70,6 +81,7 @@ mapped_pages:
 ## Operators overview [esql-operators-overview]
 
 ::::{dropdown} Operators
+:open:
 :::{include} _snippets/lists/operators.md
 :::
 ::::

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -43,14 +43,14 @@ public record BootstrapArgs(
         Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
         PathLookup pathLookup,
         Map<String, Path> sourcePaths,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(scopeResolver);
             requireNonNull(pathLookup);
             requireNonNull(sourcePaths);
-            requireNonNull(suppressFailureLogClasses);
+            requireNonNull(suppressFailureLogPackages);
         }
     }
 
@@ -78,7 +78,7 @@ public static BootstrapArgs bootstrapArgs() {
      * @param tempDir        the temp directory for Elasticsearch
      * @param logsDir        the log directory for Elasticsearch
      * @param pidFile        path to a pid file for Elasticsearch, or {@code null} if one was not specified
-     * @param suppressFailureLogClasses   classes for which we do not need or want to log Entitlements failures
+     * @param suppressFailureLogPackages   packages for which we do not need or want to log Entitlements failures
      */
     public static void bootstrap(
         Policy serverPolicyPatch,
@@ -95,7 +95,7 @@ public static void bootstrap(
         Path logsDir,
         Path tempDir,
         Path pidFile,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
@@ -119,7 +119,7 @@ public static void bootstrap(
                 settingResolver
             ),
             sourcePaths,
-            suppressFailureLogClasses
+            suppressFailureLogPackages
         );
         exportInitializationToAgent();
         loadAgent(findAgentJar());

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -90,7 +90,7 @@ private static PolicyManager createPolicyManager() {
             EntitlementBootstrap.bootstrapArgs().sourcePaths(),
             ENTITLEMENTS_MODULE,
             pathLookup,
-            bootstrapArgs.suppressFailureLogClasses()
+            bootstrapArgs.suppressFailureLogPackages()
         );
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -256,7 +256,7 @@ ModuleEntitlements policyEntitlements(String componentName, Path componentPath,
     private final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, PolicyScope> scopeResolver;
     private final PathLookup pathLookup;
-    private final Set<Class<?>> mutedClasses;
+    private final Set<Package> suppressFailureLogPackages;
 
     public static final String ALL_UNNAMED = "ALL-UNNAMED";
 
@@ -311,7 +311,7 @@ public PolicyManager(
         Map<String, Path> sourcePaths,
         Module entitlementsModule,
         PathLookup pathLookup,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy));
         this.apmAgentEntitlements = apmAgentEntitlements;
@@ -322,7 +322,7 @@ public PolicyManager(
         this.sourcePaths = sourcePaths;
         this.entitlementsModule = entitlementsModule;
         this.pathLookup = requireNonNull(pathLookup);
-        this.mutedClasses = suppressFailureLogClasses;
+        this.suppressFailureLogPackages = suppressFailureLogPackages;
 
         List<ExclusiveFileEntitlement> exclusiveFileEntitlements = new ArrayList<>();
         for (var e : serverEntitlements.entrySet()) {
@@ -688,8 +688,8 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
 
     private void notEntitled(String message, Class<?> callerClass, ModuleEntitlements entitlements) {
         var exception = new NotEntitledException(message);
-        // Don't emit a log for muted classes, e.g. classes containing self tests
-        if (mutedClasses.contains(callerClass) == false) {
+        // Don't emit a log for suppressed packages, e.g. packages containing self tests
+        if (suppressFailureLogPackages.contains(callerClass.getPackage()) == false) {
             entitlements.logger().warn("Not entitled: {}", message, exception);
         }
         throw exception;

muted-tests.yml

@@ -438,9 +438,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {lookup-join.MvJoinKeyOnTheLookupIndex ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/128030
-- class: org.elasticsearch.compute.aggregation.PercentileIntGroupingAggregatorFunctionTests
-  method: testManyInitialManyPartialFinalRunner
-  issue: https://github.com/elastic/elasticsearch/issues/128092
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test042KeystorePermissionsAreCorrect
   issue: https://github.com/elastic/elasticsearch/issues/128018
@@ -453,6 +450,18 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test041AmazonCaCertsAreInTheKeystore
   issue: https://github.com/elastic/elasticsearch/issues/128006
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test130JavaHasCorrectOwnership
+  issue: https://github.com/elastic/elasticsearch/issues/128174
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test600Interrupt
+  issue: https://github.com/elastic/elasticsearch/issues/128144
+- class: org.elasticsearch.packaging.test.EnrollmentProcessTests
+  method: test20DockerAutoFormCluster
+  issue: https://github.com/elastic/elasticsearch/issues/128113
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test121CanUseStackLoggingConfig
+  issue: https://github.com/elastic/elasticsearch/issues/128165
 
 # Examples:
 #

plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java

@@ -29,9 +29,6 @@
 import org.elasticsearch.logging.Logger;
 
 import java.io.IOException;
-import java.security.AccessController;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
 import java.util.ServiceLoader;
 
 public class AzureComputeServiceImpl extends AbstractLifecycleComponent implements AzureComputeService {
@@ -94,11 +91,8 @@ private static String getRequiredSetting(Settings settings, Setting<String> sett
     public HostedServiceGetDetailedResponse getServiceDetails() {
         SpecialPermission.check();
         try {
-            return AccessController.doPrivileged(
-                (PrivilegedExceptionAction<HostedServiceGetDetailedResponse>) () -> client.getHostedServicesOperations()
-                    .getDetailed(serviceName)
-            );
-        } catch (PrivilegedActionException e) {
+            return client.getHostedServicesOperations().getDetailed(serviceName);
+        } catch (Exception e) {
             throw new AzureServiceRemoteException("can not get list of azure nodes", e.getCause());
         }
     }

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2SeedHostsProvider.java

@@ -107,7 +107,7 @@ protected List<TransportAddress> fetchDynamicNodes() {
             // NOTE: we don't filter by security group during the describe instances request for two reasons:
             // 1. differences in VPCs require different parameters during query (ID vs Name)
             // 2. We want to use two different strategies: (all security groups vs. any security groups)
-            descInstances = SocketAccess.doPrivileged(() -> clientReference.client().describeInstances(buildDescribeInstancesRequest()));
+            descInstances = clientReference.client().describeInstances(buildDescribeInstancesRequest());
         } catch (final Exception e) {
             logger.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
             logger.debug("Full exception:", e);

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -66,7 +66,7 @@ private Ec2Client buildClient(Ec2ClientSettings clientSettings) {
             final var endpoint = Endpoint.builder().url(URI.create(clientSettings.endpoint)).build();
             ec2ClientBuilder.endpointProvider(endpointParams -> CompletableFuture.completedFuture(endpoint));
         }
-        return SocketAccess.doPrivileged(ec2ClientBuilder::build);
+        return ec2ClientBuilder.build();
     }
 
     private static void applyProxyConfiguration(Ec2ClientSettings clientSettings, ApacheHttpClient.Builder httpClientBuilder) {

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Utils.java

@@ -22,8 +22,8 @@ class AwsEc2Utils {
     static String getInstanceMetadata(String metadataPath) {
         final var httpClientBuilder = ApacheHttpClient.builder();
         httpClientBuilder.connectionTimeout(IMDS_CONNECTION_TIMEOUT);
-        try (var ec2Client = SocketAccess.doPrivileged(Ec2MetadataClient.builder().httpClient(httpClientBuilder)::build)) {
-            final var metadataValue = SocketAccess.doPrivileged(() -> ec2Client.get(metadataPath)).asString();
+        try (var ec2Client = Ec2MetadataClient.builder().httpClient(httpClientBuilder).build()) {
+            final var metadataValue = ec2Client.get(metadataPath).asString();
             if (Strings.hasText(metadataValue) == false) {
                 throw new IllegalStateException("no ec2 metadata returned from " + metadataPath);
             }