Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -48,10 +48,11 @@ public class InternalDistributionModuleCheckTaskProvider {
     /** ES jars in the lib directory that are not modularized. For now, es-log4j is the only one. */
     private static final List<String> ES_JAR_EXCLUDES = List.of("elasticsearch-log4j");
 
-    /** List of the current Elasticsearch Java Modules, by name. */
+    /** List of the current Elasticsearch Java Modules, alphabetically by name. */
     private static final List<String> EXPECTED_ES_SERVER_MODULES = List.of(
         "org.elasticsearch.base",
         "org.elasticsearch.cli",
+        "org.elasticsearch.entitlement",
         "org.elasticsearch.geo",
         "org.elasticsearch.grok",
         "org.elasticsearch.logging",

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -42,6 +42,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
 
     private Boolean debug = false;
     private Boolean cliDebug = false;
+    private Boolean entitlementsEnabled = false;
     private Boolean apmServerEnabled = false;
 
     private Boolean preserveData = false;
@@ -69,6 +70,14 @@ public void setCliDebug(boolean enabled) {
         this.cliDebug = enabled;
     }
 
+    @Option(
+        option = "entitlements",
+        description = "Use the Entitlements agent system in place of SecurityManager to enforce sandbox policies."
+    )
+    public void setEntitlementsEnabled(boolean enabled) {
+        this.entitlementsEnabled = enabled;
+    }
+
     @Input
     public Boolean getDebug() {
         return debug;
@@ -79,6 +88,11 @@ public Boolean getCliDebug() {
         return cliDebug;
     }
 
+    @Input
+    public Boolean getEntitlementsEnabled() {
+        return entitlementsEnabled;
+    }
+
     @Input
     public Boolean getApmServerEnabled() {
         return apmServerEnabled;
@@ -226,6 +240,9 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
         if (cliDebug) {
             enableCliDebug();
         }
+        if (entitlementsEnabled) {
+            enableEntitlements();
+        }
     }
 
     @TaskAction

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -74,4 +74,12 @@ default void enableCliDebug() {
             }
         }
     }
+
+    default void enableEntitlements() {
+        for (ElasticsearchCluster cluster : getClusters()) {
+            for (ElasticsearchNode node : cluster.getNodes()) {
+                node.cliJvmArgs("-Des.entitlements.enabled=true");
+            }
+        }
+    }
 }

distribution/build.gradle

@@ -262,7 +262,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
    *             Properties to expand when copying packaging files             *
    *****************************************************************************/
   configurations {
-    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each {
+    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative', 'libsEntitlementAgent', 'libsEntitlementBridge'].each {
       create(it) {
         canBeConsumed = false
         canBeResolved = true
@@ -292,6 +292,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
     libsNative project(':libs:native:native-libraries')
+    libsEntitlementAgent project(':libs:entitlement:agent')
+    libsEntitlementBridge project(':libs:entitlement:bridge')
   }
 
   project.ext {
@@ -336,6 +338,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
             include (os + '-' + architecture + '/*')
           }
         }
+        into('entitlement-agent') {
+          from(configurations.libsEntitlementAgent)
+        }
+        into('entitlement-bridge') {
+          from(configurations.libsEntitlementBridge)
+        }
       }
     }
 

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -12,18 +12,20 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 final class SystemJvmOptions {
 
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-
-        return Stream.concat(
+        boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        return Stream.of(
             Stream.of(
                 /*
                  * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property
@@ -35,8 +37,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                  * networkaddress.cache.negative ttl; set to -1 to cache forever.
                  */
                 "-Des.networkaddress.cache.negative.ttl=10",
-                // Allow to set the security manager.
-                "-Djava.security.manager=allow",
                 // pre-touch JVM emory pages during initialization
                 "-XX:+AlwaysPreTouch",
                 // explicitly set the stack size
@@ -61,15 +61,17 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
                 "-Djava.locale.providers=CLDR",
-                maybeEnableNativeAccess(),
-                maybeOverrideDockerCgroup(distroType),
-                maybeSetActiveProcessorCount(nodeSettings),
-                setReplayFile(distroType, isHotspot),
                 // Pass through distribution type
                 "-Des.distribution.type=" + distroType
             ),
-            maybeWorkaroundG1Bug()
-        ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
+            maybeEnableNativeAccess(),
+            maybeOverrideDockerCgroup(distroType),
+            maybeSetActiveProcessorCount(nodeSettings),
+            maybeSetReplayFile(distroType, isHotspot),
+            maybeWorkaroundG1Bug(),
+            maybeAllowSecurityManager(),
+            maybeAttachEntitlementAgent(useEntitlements)
+        ).flatMap(s -> s).toList();
     }
 
     /*
@@ -86,42 +88,42 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
      * that cgroup statistics are available for the container this process
      * will run in.
      */
-    private static String maybeOverrideDockerCgroup(String distroType) {
+    private static Stream<String> maybeOverrideDockerCgroup(String distroType) {
         if ("docker".equals(distroType)) {
-            return "-Des.cgroups.hierarchy.override=/";
+            return Stream.of("-Des.cgroups.hierarchy.override=/");
         }
-        return "";
+        return Stream.empty();
     }
 
-    private static String setReplayFile(String distroType, boolean isHotspot) {
+    private static Stream<String> maybeSetReplayFile(String distroType, boolean isHotspot) {
         if (isHotspot == false) {
             // the replay file option is only guaranteed for hotspot vms
-            return "";
+            return Stream.empty();
         }
         String replayDir = "logs";
         if ("rpm".equals(distroType) || "deb".equals(distroType)) {
             replayDir = "/var/log/elasticsearch";
         }
-        return "-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log";
+        return Stream.of("-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log");
     }
 
     /*
      * node.processors determines thread pool sizes for Elasticsearch. When it
      * is set, we need to also tell the JVM to respect a different value
      */
-    private static String maybeSetActiveProcessorCount(Settings nodeSettings) {
+    private static Stream<String> maybeSetActiveProcessorCount(Settings nodeSettings) {
         if (EsExecutors.NODE_PROCESSORS_SETTING.exists(nodeSettings)) {
             int allocated = EsExecutors.allocatedProcessors(nodeSettings);
-            return "-XX:ActiveProcessorCount=" + allocated;
+            return Stream.of("-XX:ActiveProcessorCount=" + allocated);
         }
-        return "";
+        return Stream.empty();
     }
 
-    private static String maybeEnableNativeAccess() {
+    private static Stream<String> maybeEnableNativeAccess() {
         if (Runtime.version().feature() >= 21) {
-            return "--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core";
+            return Stream.of("--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core");
         }
-        return "";
+        return Stream.empty();
     }
 
     /*
@@ -134,4 +136,37 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         }
         return Stream.of();
     }
+
+    private static Stream<String> maybeAllowSecurityManager() {
+        // Will become conditional on useEntitlements once entitlements can run without SM
+        return Stream.of("-Djava.security.manager=allow");
+    }
+
+    private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {
+        if (useEntitlements == false) {
+            return Stream.empty();
+        }
+
+        Path dir = Path.of("lib", "entitlement-bridge");
+        if (Files.exists(dir) == false) {
+            throw new IllegalStateException("Directory for entitlement bridge jar does not exist: " + dir);
+        }
+        String bridgeJar;
+        try (var s = Files.list(dir)) {
+            var candidates = s.limit(2).toList();
+            if (candidates.size() != 1) {
+                throw new IllegalStateException("Expected one jar in " + dir + "; found " + candidates.size());
+            }
+            bridgeJar = candidates.get(0).toString();
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
+        }
+        return Stream.of(
+            "-Des.entitlements.enabled=true",
+            "-XX:+EnableDynamicAgentLoading",
+            "-Djdk.attach.allowAttachSelf=true",
+            "--patch-module=java.base=" + bridgeJar,
+            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement"
+        );
+    }
 }

docs/changelog/114879.yaml

@@ -0,0 +1,5 @@
+pr: 114879
+summary: Add refresh `.security` index call between security migrations
+area: Security
+type: enhancement
+issues: []

docs/changelog/115876.yaml

@@ -0,0 +1,5 @@
+pr: 115876
+summary: Inference duration and error metrics
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/116082.yaml

@@ -0,0 +1,5 @@
+pr: 116082
+summary: Add support for bitwise inner-product in painless
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/116128.yaml

@@ -0,0 +1,5 @@
+pr: 116128
+summary: Add num docs and size to logsdb telemetry
+area: Logs
+type: enhancement
+issues: []

docs/changelog/116174.yaml

@@ -0,0 +1,7 @@
+pr: 116174
+summary: Handle with `illegalArgumentExceptions` negative values in HDR percentile
+  aggregations
+area: Aggregations
+type: bug
+issues:
+ - 115777