Merge branch 'main' into lucene_snapshot

Author: ChrisHegarty

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java

@@ -193,7 +193,7 @@ public void testStdinInputWithCarriageReturn() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        terminal.addSecretInput("Typedthisandhitenter\r");
+        terminal.addSecretInput("Typedthisandhitenter\r\n");
         execute("-x", "foo");
         assertSecureString("foo", "Typedthisandhitenter", password);
     }

docs/changelog/126397.yaml

@@ -0,0 +1,6 @@
+pr: 126397
+summary: "ESQL: Preserve single aggregate when all attributes are pruned"
+area: ES|QL
+type: bug
+issues:
+ - 126392

docs/changelog/126653.yaml

@@ -0,0 +1,5 @@
+pr: 126653
+summary: Retry shard movements during ESQL query
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/mapping-date-format.md

@@ -27,7 +27,7 @@ Many APIs which support date values also support [date math](/reference/elastics
 
 ## Custom date formats [custom-date-formats]
 
-Completely customizable date formats are supported. The syntax for these is explained in [DateTimeFormatter docs](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/time/format/DateTimeFormatter.md).
+Completely customizable date formats are supported. The syntax for these is explained in [DateTimeFormatter docs](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/time/format/DateTimeFormatter.html).
 
 Note that while the built-in formats for week dates use the ISO definition of weekyears, custom formatters using the `Y`, `W`, or `w` field specifiers use the JDK locale definition of weekyears. This can result in different values between the built-in formats and custom formats for week dates.
 

docs/release-notes/known-issues.md

@@ -7,3 +7,23 @@ mapped_pages:
 # Elasticsearch known issues [elasticsearch-known-issues]
 Known issues are significant defects or limitations that may impact your implementation. These issues are actively being worked on and will be addressed in a future release. Review the Elasticsearch known issues to help you make informed decisions, such as upgrading to a new version.
 
+## 9.0.0 [elasticsearch-9.0.0-known-issues]
+* Elasticsearch on Windows might fail to start, or might forbid some file-related operations, when referencing paths with a case different from the one stored by the filesystem. Windows treats paths as case-insensitive, but the filesystem stores them with case. Entitlements, the new security system used by Elasticsearch, treat all paths as case-sensitive, and can therefore prevent access to a path that should be accessible.
+
+  For example: If Elasticsearch is installed in  `C:\ELK\elasticsearch`, and you try to launch it as `c:\elk\elasticsearch\bin\elasticsearch.bat`, you will get a `NotEntitledException` while booting. This is because Elasticsearch blocks access to `c:\elk\elasticsearch`, because does not match `C:\ELK\elasticsearch`. \
+This issue will be fixed in a future patch release (see [PR #126990](https://github.com/elastic/elasticsearch/pull/126990)).
+
+  As a workaround, make sure that all paths you specify have the same casing as the paths stored in the filesystem. Files and directory names should be entered as they appear in Windows Explorer or in a command prompt. This applies to paths specified in the command line, config files, environment variables and secure settings.
+
+* Active Directory authentication is blocked by default. Entitlements, the new security system used by Elasticsearch, has a policy for the `x-pack-core` module that is too restrictive, and does not allow the LDAP library used for AD authentication to perform outbound network connections. This issue will be fixed in a future patch release (see [PR #126992](https://github.com/elastic/elasticsearch/pull/126992)).
+
+  As a workaround, you can temporarily patch the policy using a JVM option:
+
+  1. Create a file called `${ES_CONF_PATH}/jvm_options/workaround-127061.options`.
+  2. Add the following line to the new file:
+
+     ```
+     -Des.entitlements.policy.x-pack-core=dmVyc2lvbnM6CiAgLSA4LjE4LjAKICAtIDkuMC4wCnBvbGljeToKICB1bmJvdW5kaWQubGRhcHNkazoKICAgIC0gc2V0X2h0dHBzX2Nvbm5lY3Rpb25fcHJvcGVydGllcwogICAgLSBvdXRib3VuZF9uZXR3b3Jr
+     ```
+
+  For information about editing your JVM settings, refer to [JVM settings](https://www.elastic.co/docs/reference/elasticsearch/jvm-settings).

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/package-info.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * Contains classes that need to be used directly from instrumented methods.
+ * It's a minimal shim that is patched into the {@code java.base} module so that it is callable from the class library methods instrumented
+ * by the agent. The shim retains a {@link org.elasticsearch.entitlement.bridge.EntitlementChecker} instance (inside its
+ * {@link org.elasticsearch.entitlement.bridge.EntitlementCheckerHandle} holder) and forwards the entitlement checks to the main library,
+ * that exists in the system classloader.
+ * {@link org.elasticsearch.entitlement.bridge.EntitlementChecker} holds all the entitlements check definitions, one for each instrumented
+ * method.
+ * <p>
+ * In order to work across multiple Java versions, this project uses multi-release jars via the {@code mrjar} plugin, which makes it is
+ * possible to specify classes for specific Java versions in specific {@code src} folders (e.g. {@code main23} for classes available to
+ * Java 23+).
+ * All the versioned Java classes are merged into the bridge jar. Therefore, we must prefix the class name
+ * with the version, e.g. {@code Java23EntitlementCheckerHandle} and {@code Java23EntitlementChecker}.
+ * </p>
+ */
+package org.elasticsearch.entitlement.bridge;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -15,9 +15,13 @@
 import com.sun.tools.attach.VirtualMachine;
 
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
+import org.elasticsearch.entitlement.runtime.policy.PathLookup;
+import org.elasticsearch.entitlement.runtime.policy.PathLookupImpl;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
+import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
@@ -36,36 +40,16 @@ public class EntitlementBootstrap {
     public record BootstrapArgs(
         @Nullable Policy serverPolicyPatch,
         Map<String, Policy> pluginPolicies,
-        Function<Class<?>, String> pluginResolver,
-        Function<String, Stream<String>> settingResolver,
-        Path[] dataDirs,
-        Path[] sharedRepoDirs,
-        Path configDir,
-        Path libDir,
-        Path modulesDir,
-        Path pluginsDir,
+        Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
+        PathLookup pathLookup,
         Map<String, Path> sourcePaths,
-        Path logsDir,
-        Path tempDir,
-        Path pidFile,
         Set<Class<?>> suppressFailureLogClasses
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
-            requireNonNull(pluginResolver);
-            requireNonNull(settingResolver);
-            requireNonNull(dataDirs);
-            if (dataDirs.length == 0) {
-                throw new IllegalArgumentException("must provide at least one data directory");
-            }
-            requireNonNull(sharedRepoDirs);
-            requireNonNull(configDir);
-            requireNonNull(libDir);
-            requireNonNull(modulesDir);
-            requireNonNull(pluginsDir);
+            requireNonNull(scopeResolver);
+            requireNonNull(pathLookup);
             requireNonNull(sourcePaths);
-            requireNonNull(logsDir);
-            requireNonNull(tempDir);
             requireNonNull(suppressFailureLogClasses);
         }
     }
@@ -82,7 +66,7 @@ public static BootstrapArgs bootstrapArgs() {
      *
      * @param serverPolicyPatch a policy with additional entitlements to patch the embedded server layer policy
      * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
-     * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
+     * @param scopeResolver a functor to map a Java Class to the component and module it belongs to.
      * @param settingResolver a functor to resolve a setting name pattern for one or more Elasticsearch settings.
      * @param dataDirs       data directories for Elasticsearch
      * @param sharedRepoDirs shared repository directories for Elasticsearch
@@ -99,7 +83,7 @@ public static BootstrapArgs bootstrapArgs() {
     public static void bootstrap(
         Policy serverPolicyPatch,
         Map<String, Policy> pluginPolicies,
-        Function<Class<?>, String> pluginResolver,
+        Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
         Function<String, Stream<String>> settingResolver,
         Path[] dataDirs,
         Path[] sharedRepoDirs,
@@ -120,24 +104,35 @@ public static void bootstrap(
         EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(
             serverPolicyPatch,
             pluginPolicies,
-            pluginResolver,
-            settingResolver,
-            dataDirs,
-            sharedRepoDirs,
-            configDir,
-            libDir,
-            modulesDir,
-            pluginsDir,
+            scopeResolver,
+            new PathLookupImpl(
+                getUserHome(),
+                configDir,
+                dataDirs,
+                sharedRepoDirs,
+                libDir,
+                modulesDir,
+                pluginsDir,
+                logsDir,
+                tempDir,
+                pidFile,
+                settingResolver
+            ),
             sourcePaths,
-            logsDir,
-            tempDir,
-            pidFile,
             suppressFailureLogClasses
         );
         exportInitializationToAgent();
         loadAgent(findAgentJar());
     }
 
+    private static Path getUserHome() {
+        String userHome = System.getProperty("user.home");
+        if (userHome == null) {
+            throw new IllegalStateException("user.home system property is required");
+        }
+        return PathUtils.get(userHome);
+    }
+
     @SuppressForbidden(reason = "The VirtualMachine API is the only way to attach a java agent dynamically")
     private static void loadAgent(String agentPath) {
         try {