Merge remote-tracking branch 'origin/main' into improve-memory-accounting

Author: fcofdez

benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
@@ -84,7 +85,11 @@ public void setUp() {
         }
         int mid = indices.length / 2;
         clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
-        resolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(List.of()));
+        resolver = new IndexNameExpressionResolver(
+            new ThreadContext(Settings.EMPTY),
+            new SystemIndices(List.of()),
+            DefaultProjectResolver.INSTANCE
+        );
         indexListRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), indices);
         starRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), "*");
         String[] mixed = indices.clone();

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -138,7 +138,7 @@ public void setUp() throws Exception {
         Metadata metadata = mb.build();
         RoutingTable.Builder rb = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 1; i <= numIndices; i++) {
-            rb.addAsNew(metadata.index("test_" + i));
+            rb.addAsNew(metadata.getProject().index("test_" + i));
         }
         RoutingTable routingTable = rb.build();
         DiscoveryNodes.Builder nb = DiscoveryNodes.builder();

docs/changelog/123492.yaml

@@ -0,0 +1,6 @@
+pr: 123492
+summary: Fix function registry concurrency issues on constructor
+area: ES|QL
+type: bug
+issues:
+ - 123430

docs/changelog/123569.yaml

@@ -0,0 +1,5 @@
+pr: 123569
+summary: Abort pending deletion on `IndicesService` close
+area: Store
+type: enhancement
+issues: []

libs/core/src/main/java/org/elasticsearch/core/FixForMultiProject.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.core;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to identify a block of code (a whole class, a method, a field, or a local variable) that needs to be reviewed (for cleanup,
+ * removal or change) before merging the multi-project branch into main
+ */
+@Retention(RetentionPolicy.SOURCE)
+@Target(
+    { ElementType.LOCAL_VARIABLE, ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.MODULE }
+)
+public @interface FixForMultiProject {
+
+    /**
+     * Some explanation on what and why for the future fix
+     */
+    String description() default "";
+}

libs/core/src/main/java/org/elasticsearch/core/Releasables.java

@@ -9,8 +9,6 @@
 
 package org.elasticsearch.core;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicReference;
@@ -21,11 +19,17 @@ public enum Releasables {
 
     /** Release the provided {@link Releasable}s. */
     public static void close(Iterable<? extends Releasable> releasables) {
-        try {
-            // this does the right thing with respect to add suppressed and not wrapping errors etc.
-            IOUtils.close(releasables);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
+        RuntimeException firstException = null;
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                firstException = useOrSuppress(firstException, e);
+            }
+        }
+
+        if (firstException != null) {
+            throw firstException;
         }
     }
 
@@ -38,7 +42,18 @@ public static void close(@Nullable Releasable releasable) {
 
     /** Release the provided {@link Releasable}s. */
     public static void close(Releasable... releasables) {
-        close(true, releasables);
+        RuntimeException firstException = null;
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                firstException = useOrSuppress(firstException, e);
+            }
+        }
+
+        if (firstException != null) {
+            throw firstException;
+        }
     }
 
     /** Release the provided {@link Releasable}s expecting no exception to by thrown by any of them. */
@@ -63,19 +78,21 @@ public static void closeExpectNoException(Releasable releasable) {
 
     /** Release the provided {@link Releasable}s, ignoring exceptions. */
     public static void closeWhileHandlingException(Releasable... releasables) {
-        close(false, releasables);
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                // ignored
+            }
+        }
     }
 
-    /** Release the provided {@link Releasable}s, ignoring exceptions if <code>success</code> is {@code false}. */
-    private static void close(boolean success, Releasable... releasables) {
-        try {
-            // this does the right thing with respect to add suppressed and not wrapping errors etc.
-            IOUtils.close(releasables);
-        } catch (IOException e) {
-            if (success) {
-                throw new UncheckedIOException(e);
-            }
+    private static RuntimeException useOrSuppress(RuntimeException firstException, RuntimeException e) {
+        if (firstException == null || firstException == e) {
+            return e;
         }
+        firstException.addSuppressed(e);
+        return firstException;
     }
 
     /** Wrap several releasables into a single one. This is typically useful for use with try-with-resources: for example let's assume

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement;
 
 import java.lang.instrument.Instrumentation;
 import java.lang.reflect.Constructor;
@@ -242,7 +243,14 @@ private static PolicyManager createPolicyManager() {
         if (trustStorePath != null) {
             Collections.addAll(
                 serverScopes,
-                new Scope("org.bouncycastle.fips.tls", List.of(new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))))),
+                new Scope(
+                    "org.bouncycastle.fips.tls",
+                    List.of(
+                        new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))),
+                        new OutboundNetworkEntitlement(),
+                        new ManageThreadsEntitlement()
+                    )
+                ),
                 new Scope(
                     "org.bouncycastle.fips.core",
                     // read to lib dir is required for checksum validation
@@ -258,6 +266,8 @@ private static PolicyManager createPolicyManager() {
         List<Entitlement> agentEntitlements = List.of(
             new CreateClassLoaderEntitlement(),
             new ManageThreadsEntitlement(),
+            new SetHttpsConnectionPropertiesEntitlement(),
+            new OutboundNetworkEntitlement(),
             new FilesEntitlement(
                 List.of(
                     FileData.ofPath(Path.of("/co/elastic/apm/agent/"), READ),

libs/ssl-config/build.gradle

@@ -10,6 +10,7 @@ apply plugin: "elasticsearch.publish"
 
 dependencies {
   api project(':libs:core')
+  api project(':libs:entitlement')
 
   testImplementation(project(":test:framework")) {
     exclude group: 'org.elasticsearch', module: 'ssl-config'

libs/ssl-config/src/main/java/module-info.java

@@ -9,6 +9,7 @@
 
 module org.elasticsearch.sslconfig {
     requires org.elasticsearch.base;
+    requires org.elasticsearch.entitlement;
 
     exports org.elasticsearch.common.ssl;
 }

libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.common.ssl;
 
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -127,6 +128,8 @@ private PrivateKey getPrivateKey(Path path) {
             return privateKey;
         } catch (AccessControlException e) {
             throw SslFileUtil.accessControlFailure(KEY_FILE_TYPE, List.of(path), e, configBasePath);
+        } catch (NotEntitledException e) {
+            throw SslFileUtil.notEntitledFailure(KEY_FILE_TYPE, List.of(path), e, configBasePath);
         } catch (IOException e) {
             throw SslFileUtil.ioException(KEY_FILE_TYPE, List.of(path), e);
         } catch (GeneralSecurityException e) {