Merge remote-tracking branch 'elastic/main' into batched-exec-short

Author: original-brownbear

.buildkite/pipelines/periodic-fwc.yml

@@ -1,7 +1,7 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -13,4 +13,4 @@ steps:
       setup:
         FWC_VERSION: []
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
@@ -84,7 +85,11 @@ public void setUp() {
         }
         int mid = indices.length / 2;
         clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
-        resolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(List.of()));
+        resolver = new IndexNameExpressionResolver(
+            new ThreadContext(Settings.EMPTY),
+            new SystemIndices(List.of()),
+            DefaultProjectResolver.INSTANCE
+        );
         indexListRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), indices);
         starRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), "*");
         String[] mixed = indices.clone();

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -138,7 +138,7 @@ public void setUp() throws Exception {
         Metadata metadata = mb.build();
         RoutingTable.Builder rb = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 1; i <= numIndices; i++) {
-            rb.addAsNew(metadata.index("test_" + i));
+            rb.addAsNew(metadata.getProject().index("test_" + i));
         }
         RoutingTable routingTable = rb.build();
         DiscoveryNodes.Builder nb = DiscoveryNodes.builder();

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -145,18 +145,23 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     }
   }
 
-  // modifies the idea module config to enable preview features on ':libs:native' module
+  // modifies the idea module config to enable preview features on module that need them
   tasks.register("enablePreviewFeatures", EnablePreviewFeaturesTask) {
     group = 'ide'
-    description = 'Enables preview features on native library module'
+    description = 'Enables preview features on modules that need them'
     dependsOn tasks.named("enableExternalConfiguration")
     doLast {
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW')
+      // due to org.elasticsearch.plugins.PluginsLoader
+      enablePreview('.idea/modules/server/elasticsearch.server.main.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/server/elasticsearch.server.test.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/elasticsearch.libs.entitlement.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/elasticsearch.libs.entitlement.test.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/bridge/elasticsearch.libs.entitlement.bridge.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/bridge/elasticsearch.libs.entitlement.bridge.test.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/libs/entitlement/qa/entitlement-test-plugin/elasticsearch.libs.entitlement.qa.entitlement-test-plugin.main.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/libs/entitlement/qa/entitlement-test-plugin/elasticsearch.libs.entitlement.qa.entitlement-test-plugin.test.iml', 'JDK_21_PREVIEW')
     }
   }
 

docs/changelog/123492.yaml

@@ -0,0 +1,6 @@
+pr: 123492
+summary: Fix function registry concurrency issues on constructor
+area: ES|QL
+type: bug
+issues:
+ - 123430

libs/core/src/main/java/org/elasticsearch/core/FixForMultiProject.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.core;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to identify a block of code (a whole class, a method, a field, or a local variable) that needs to be reviewed (for cleanup,
+ * removal or change) before merging the multi-project branch into main
+ */
+@Retention(RetentionPolicy.SOURCE)
+@Target(
+    { ElementType.LOCAL_VARIABLE, ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.MODULE }
+)
+public @interface FixForMultiProject {
+
+    /**
+     * Some explanation on what and why for the future fix
+     */
+    String description() default "";
+}

libs/core/src/main/java/org/elasticsearch/core/Releasables.java

@@ -9,8 +9,6 @@
 
 package org.elasticsearch.core;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicReference;
@@ -21,11 +19,17 @@ public enum Releasables {
 
     /** Release the provided {@link Releasable}s. */
     public static void close(Iterable<? extends Releasable> releasables) {
-        try {
-            // this does the right thing with respect to add suppressed and not wrapping errors etc.
-            IOUtils.close(releasables);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
+        RuntimeException firstException = null;
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                firstException = useOrSuppress(firstException, e);
+            }
+        }
+
+        if (firstException != null) {
+            throw firstException;
         }
     }
 
@@ -38,7 +42,18 @@ public static void close(@Nullable Releasable releasable) {
 
     /** Release the provided {@link Releasable}s. */
     public static void close(Releasable... releasables) {
-        close(true, releasables);
+        RuntimeException firstException = null;
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                firstException = useOrSuppress(firstException, e);
+            }
+        }
+
+        if (firstException != null) {
+            throw firstException;
+        }
     }
 
     /** Release the provided {@link Releasable}s expecting no exception to by thrown by any of them. */
@@ -63,19 +78,21 @@ public static void closeExpectNoException(Releasable releasable) {
 
     /** Release the provided {@link Releasable}s, ignoring exceptions. */
     public static void closeWhileHandlingException(Releasable... releasables) {
-        close(false, releasables);
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                // ignored
+            }
+        }
     }
 
-    /** Release the provided {@link Releasable}s, ignoring exceptions if <code>success</code> is {@code false}. */
-    private static void close(boolean success, Releasable... releasables) {
-        try {
-            // this does the right thing with respect to add suppressed and not wrapping errors etc.
-            IOUtils.close(releasables);
-        } catch (IOException e) {
-            if (success) {
-                throw new UncheckedIOException(e);
-            }
+    private static RuntimeException useOrSuppress(RuntimeException firstException, RuntimeException e) {
+        if (firstException == null || firstException == e) {
+            return e;
         }
+        firstException.addSuppressed(e);
+        return firstException;
     }
 
     /** Wrap several releasables into a single one. This is typically useful for use with try-with-resources: for example let's assume

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement;
 
 import java.lang.instrument.Instrumentation;
 import java.lang.reflect.Constructor;
@@ -242,7 +243,14 @@ private static PolicyManager createPolicyManager() {
         if (trustStorePath != null) {
             Collections.addAll(
                 serverScopes,
-                new Scope("org.bouncycastle.fips.tls", List.of(new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))))),
+                new Scope(
+                    "org.bouncycastle.fips.tls",
+                    List.of(
+                        new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))),
+                        new OutboundNetworkEntitlement(),
+                        new ManageThreadsEntitlement()
+                    )
+                ),
                 new Scope(
                     "org.bouncycastle.fips.core",
                     // read to lib dir is required for checksum validation
@@ -258,6 +266,8 @@ private static PolicyManager createPolicyManager() {
         List<Entitlement> agentEntitlements = List.of(
             new CreateClassLoaderEntitlement(),
             new ManageThreadsEntitlement(),
+            new SetHttpsConnectionPropertiesEntitlement(),
+            new OutboundNetworkEntitlement(),
             new FilesEntitlement(
                 List.of(
                     FileData.ofPath(Path.of("/co/elastic/apm/agent/"), READ),

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -30,13 +30,74 @@
 
 public final class FileAccessTree {
 
+    /**
+     * An intermediary structure to help build exclusive paths for files entitlements.
+     */
+    record ExclusiveFileEntitlement(String componentName, String moduleName, FilesEntitlement filesEntitlement) {}
+
+    /**
+     * An intermediary structure to help globally validate exclusive paths, and then build exclusive paths for individual modules.
+     */
+    record ExclusivePath(String componentName, String moduleName, String path) {
+
+        @Override
+        public String toString() {
+            return "[[" + componentName + "] [" + moduleName + "] [" + path + "]]";
+        }
+    }
+
+    static List<ExclusivePath> buildExclusivePathList(List<ExclusiveFileEntitlement> exclusiveFileEntitlements, PathLookup pathLookup) {
+        List<ExclusivePath> exclusivePaths = new ArrayList<>();
+        for (ExclusiveFileEntitlement efe : exclusiveFileEntitlements) {
+            for (FilesEntitlement.FileData fd : efe.filesEntitlement().filesData()) {
+                if (fd.exclusive()) {
+                    List<Path> paths = fd.resolvePaths(pathLookup).toList();
+                    for (Path path : paths) {
+                        exclusivePaths.add(new ExclusivePath(efe.componentName(), efe.moduleName(), normalizePath(path)));
+                    }
+                }
+            }
+        }
+        exclusivePaths.sort((ep1, ep2) -> PATH_ORDER.compare(ep1.path(), ep2.path()));
+        return exclusivePaths;
+    }
+
+    static void validateExclusivePaths(List<ExclusivePath> exclusivePaths) {
+        if (exclusivePaths.isEmpty() == false) {
+            ExclusivePath currentExclusivePath = exclusivePaths.get(0);
+            for (int i = 1; i < exclusivePaths.size(); ++i) {
+                ExclusivePath nextPath = exclusivePaths.get(i);
+                if (currentExclusivePath.path().equals(nextPath.path) || isParent(currentExclusivePath.path(), nextPath.path())) {
+                    throw new IllegalArgumentException(
+                        "duplicate/overlapping exclusive paths found in files entitlements: " + currentExclusivePath + " and " + nextPath
+                    );
+                }
+                currentExclusivePath = nextPath;
+            }
+        }
+    }
+
     private static final Logger logger = LogManager.getLogger(FileAccessTree.class);
     private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator();
 
+    private final String[] exclusivePaths;
     private final String[] readPaths;
     private final String[] writePaths;
 
-    private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
+    private FileAccessTree(
+        String componentName,
+        String moduleName,
+        FilesEntitlement filesEntitlement,
+        PathLookup pathLookup,
+        List<ExclusivePath> exclusivePaths
+    ) {
+        List<String> updatedExclusivePaths = new ArrayList<>();
+        for (ExclusivePath exclusivePath : exclusivePaths) {
+            if (exclusivePath.componentName().equals(componentName) == false || exclusivePath.moduleName().equals(moduleName) == false) {
+                updatedExclusivePaths.add(exclusivePath.path());
+            }
+        }
+
         List<String> readPaths = new ArrayList<>();
         List<String> writePaths = new ArrayList<>();
         BiConsumer<Path, Mode> addPath = (path, mode) -> {
@@ -83,9 +144,11 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         Path jdk = Paths.get(System.getProperty("java.home"));
         addPathAndMaybeLink.accept(jdk.resolve("conf"), Mode.READ);
 
+        updatedExclusivePaths.sort(PATH_ORDER);
         readPaths.sort(PATH_ORDER);
         writePaths.sort(PATH_ORDER);
 
+        this.exclusivePaths = updatedExclusivePaths.toArray(new String[0]);
         this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
         this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
     }
@@ -106,8 +169,14 @@ private static List<String> pruneSortedPaths(List<String> paths) {
         return prunedReadPaths;
     }
 
-    public static FileAccessTree of(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
-        return new FileAccessTree(filesEntitlement, pathLookup);
+    public static FileAccessTree of(
+        String componentName,
+        String moduleName,
+        FilesEntitlement filesEntitlement,
+        PathLookup pathLookup,
+        List<ExclusivePath> exclusivePaths
+    ) {
+        return new FileAccessTree(componentName, moduleName, filesEntitlement, pathLookup, exclusivePaths);
     }
 
     boolean canRead(Path path) {
@@ -132,10 +201,16 @@ static String normalizePath(Path path) {
         return result;
     }
 
-    private static boolean checkPath(String path, String[] paths) {
+    private boolean checkPath(String path, String[] paths) {
         if (paths.length == 0) {
             return false;
         }
+
+        int endx = Arrays.binarySearch(exclusivePaths, path, PATH_ORDER);
+        if (endx < -1 && isParent(exclusivePaths[-endx - 2], path) || endx >= 0) {
+            return false;
+        }
+
         int ndx = Arrays.binarySearch(paths, path, PATH_ORDER);
         if (ndx < -1) {
             return isParent(paths[-ndx - 2], path);