Merge branch 'main' into ivf_centroidRaw

Author: iverase

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -34,6 +34,7 @@
 import java.io.File;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Stream;
 
 import javax.inject.Inject;
@@ -50,6 +51,8 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
 
     public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
 
+    public static final Set<String> TEST_TASKS_WITH_ENTITLEMENTS = Set.of("test", "internalClusterTest");
+
     @Inject
     protected abstract ProviderFactory getProviderFactory();
 
@@ -174,14 +177,23 @@ public void execute(Task t) {
             nonInputProperties.systemProperty("workspace.dir", Util.locateElasticsearchWorkspace(project.getGradle()));
             // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
             nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
+            if (test.getName().equals("internalClusterTest")) {
+                // configure a node home directory independent of the Java temp dir so that entitlements can be properly enforced
+                nonInputProperties.systemProperty("tempDir", test.getWorkingDir().toPath().resolve("nodesTemp"));
+            }
 
             SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
             SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
             SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
-            if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
+            SourceSet internalClusterTestSourceSet = sourceSets.findByName("internalClusterTest");
+
+            if (TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) && mainSourceSet != null && testSourceSet != null) {
                 FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
                 FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
-                FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
+                FileCollection internalClusterTestRuntime = ("internalClusterTest".equals(test.getName())
+                    && internalClusterTestSourceSet != null) ? internalClusterTestSourceSet.getRuntimeClasspath() : project.files();
+                FileCollection testOnlyFiles = testRuntime.plus(internalClusterTestRuntime).minus(mainRuntime);
+
                 test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
             }
 
@@ -241,14 +253,15 @@ public void execute(Task t) {
      * Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
      */
     private void configureJavaBaseModuleOptions(Project project) {
-        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
-            FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            // patch immutable collections only for "test" task
+            FileCollection patchedImmutableCollections = test.getName().equals("test") ? patchedImmutableCollections(project) : null;
             if (patchedImmutableCollections != null) {
                 test.getInputs().files(patchedImmutableCollections);
                 test.systemProperty("tests.hackImmutableCollections", "true");
             }
 
-            FileCollection entitlementBridge = entitlementBridge(project);
+            FileCollection entitlementBridge = TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) ? entitlementBridge(project) : null;
             if (entitlementBridge != null) {
                 test.getInputs().files(entitlementBridge);
             }
@@ -312,27 +325,30 @@ private static void configureEntitlements(Project project) {
         }
         FileCollection bridgeFiles = bridgeConfig;
 
-        project.getTasks().withType(Test.class).configureEach(test -> {
-            // See also SystemJvmOptions.maybeAttachEntitlementAgent.
-
-            // Agent
-            if (agentFiles.isEmpty() == false) {
-                test.getInputs().files(agentFiles);
-                test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
-                test.systemProperty("jdk.attach.allowAttachSelf", true);
-            }
+        project.getTasks()
+            .withType(Test.class)
+            .matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
+            .configureEach(test -> {
+                // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+
+                // Agent
+                if (agentFiles.isEmpty() == false) {
+                    test.getInputs().files(agentFiles);
+                    test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
+                    test.systemProperty("jdk.attach.allowAttachSelf", true);
+                }
 
-            // Bridge
-            if (bridgeFiles.isEmpty() == false) {
-                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
-                test.getInputs().files(bridgeFiles);
-                // Tests may not be modular, but the JDK still is
-                test.jvmArgs(
-                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
-                        + modulesContainingEntitlementInstrumentation
-                );
-            }
-        });
+                // Bridge
+                if (bridgeFiles.isEmpty() == false) {
+                    String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                    test.getInputs().files(bridgeFiles);
+                    // Tests may not be modular, but the JDK still is
+                    test.jvmArgs(
+                        "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                            + modulesContainingEntitlementInstrumentation
+                    );
+                }
+            });
     }
 
 }

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -58,9 +58,12 @@ public void apply(Project project) {
         });
 
         if (project.getRootProject().getName().equals("elasticsearch")) {
-            project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
-                test.systemProperty("es.entitlement.enableForTests", "true");
-            });
+            project.getTasks()
+                .withType(Test.class)
+                .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
+                .configureEach(test -> {
+                    test.systemProperty("es.entitlement.enableForTests", "true");
+                });
         }
     }
 }

docs/changelog/129848.yaml

@@ -0,0 +1,5 @@
+pr: 129848
+summary: "[ML] Add Azure AI Rerank support to the Inference Plugin"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/131391.yaml

@@ -0,0 +1,6 @@
+pr: 131391
+summary: Fix bug in point in time response
+area: Search
+type: bug
+issues:
+ - 131026

libs/build.gradle

@@ -50,7 +50,7 @@ configure(childProjects.values()) {
   // Omit oddball libraries that aren't in server.
   def nonServerLibs = ['plugin-scanner']
   if (false == nonServerLibs.contains(project.name)) {
-    project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
+    project.getTasks().withType(Test.class).matching(test -> ['test', 'internalClusterTest'].contains(test.name)).configureEach(test -> {
       test.systemProperty('es.entitlement.enableForTests', 'true')
     })
   }

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java renamed to modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsTests.java

@@ -57,7 +57,7 @@
                     // 'WindowsFS.checkDeleteAccess(...)').
     }
 )
-public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase {
+public class ReloadingDatabasesWhilePerformingGeoLookupsTests extends ESTestCase {
 
     /**
      * This tests essentially verifies that a Maxmind database reader doesn't fail with:

modules/repository-s3/src/main/resources/org/elasticsearch/repositories/s3/regions_by_endpoint.txt

@@ -6,6 +6,10 @@ ap-east-1          s3-fips.ap-east-1.amazonaws.com
 ap-east-1          s3-fips.dualstack.ap-east-1.amazonaws.com
 ap-east-1          s3.ap-east-1.amazonaws.com
 ap-east-1          s3.dualstack.ap-east-1.amazonaws.com
+ap-east-2          s3-fips.ap-east-2.amazonaws.com
+ap-east-2          s3-fips.dualstack.ap-east-2.amazonaws.com
+ap-east-2          s3.ap-east-2.amazonaws.com
+ap-east-2          s3.dualstack.ap-east-2.amazonaws.com
 ap-northeast-1     s3-fips.ap-northeast-1.amazonaws.com
 ap-northeast-1     s3-fips.dualstack.ap-northeast-1.amazonaws.com
 ap-northeast-1     s3.ap-northeast-1.amazonaws.com
@@ -56,6 +60,14 @@ aws-iso-b-global   s3-fips.aws-iso-b-global.sc2s.sgov.gov
 aws-iso-b-global   s3-fips.dualstack.aws-iso-b-global.sc2s.sgov.gov
 aws-iso-b-global   s3.aws-iso-b-global.sc2s.sgov.gov
 aws-iso-b-global   s3.dualstack.aws-iso-b-global.sc2s.sgov.gov
+aws-iso-e-global   s3-fips.aws-iso-e-global.cloud.adc-e.uk
+aws-iso-e-global   s3-fips.dualstack.aws-iso-e-global.cloud.adc-e.uk
+aws-iso-e-global   s3.aws-iso-e-global.cloud.adc-e.uk
+aws-iso-e-global   s3.dualstack.aws-iso-e-global.cloud.adc-e.uk
+aws-iso-f-global   s3-fips.aws-iso-f-global.csp.hci.ic.gov
+aws-iso-f-global   s3-fips.dualstack.aws-iso-f-global.csp.hci.ic.gov
+aws-iso-f-global   s3.aws-iso-f-global.csp.hci.ic.gov
+aws-iso-f-global   s3.dualstack.aws-iso-f-global.csp.hci.ic.gov
 aws-iso-global     s3-fips.aws-iso-global.c2s.ic.gov
 aws-iso-global     s3-fips.dualstack.aws-iso-global.c2s.ic.gov
 aws-iso-global     s3.aws-iso-global.c2s.ic.gov
@@ -76,6 +88,10 @@ cn-north-1         s3.cn-north-1.amazonaws.com.cn
 cn-north-1         s3.dualstack.cn-north-1.amazonaws.com.cn
 cn-northwest-1     s3.cn-northwest-1.amazonaws.com.cn
 cn-northwest-1     s3.dualstack.cn-northwest-1.amazonaws.com.cn
+eusc-de-east-1     s3-fips.eusc-de-east-1.amazonaws.eu
+eusc-de-east-1     s3-fips.dualstack.eusc-de-east-1.amazonaws.eu
+eusc-de-east-1     s3.eusc-de-east-1.amazonaws.eu
+eusc-de-east-1     s3.dualstack.eusc-de-east-1.amazonaws.eu
 eu-central-1       s3-fips.dualstack.eu-central-1.amazonaws.com
 eu-central-1       s3-fips.eu-central-1.amazonaws.com
 eu-central-1       s3.dualstack.eu-central-1.amazonaws.com

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RegionFromEndpointGuesserTests.java

@@ -9,6 +9,11 @@
 
 package org.elasticsearch.repositories.s3;
 
+import software.amazon.awssdk.endpoints.Endpoint;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.endpoints.S3EndpointParams;
+import software.amazon.awssdk.services.s3.endpoints.internal.DefaultS3EndpointProvider;
+
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.test.ESTestCase;
 
@@ -23,6 +28,14 @@ public void testRegionGuessing() {
         assertRegionGuess("random.endpoint.internal.net", null);
     }
 
+    public void testHasEntryForEachRegion() {
+        final var defaultS3EndpointProvider = new DefaultS3EndpointProvider();
+        for (var region : Region.regions()) {
+            final Endpoint endpoint = safeGet(defaultS3EndpointProvider.resolveEndpoint(S3EndpointParams.builder().region(region).build()));
+            assertNotNull(region.id(), RegionFromEndpointGuesser.guessRegion(endpoint.url().toString()));
+        }
+    }
+
     private static void assertRegionGuess(String endpoint, @Nullable String expectedRegion) {
         assertEquals(endpoint, expectedRegion, RegionFromEndpointGuesser.guessRegion(endpoint));
     }

server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 
 import java.io.IOException;
@@ -39,6 +40,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+@ESTestCase.WithoutEntitlements // CLI tools don't run with entitlements enforced
 public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase {
 
     private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort) throws Exception {

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -341,6 +341,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion LOOKUP_JOIN_CCS = def(9_120_0_00);
     public static final TransportVersion NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO = def(9_121_0_00);
     public static final TransportVersion ESQL_CATEGORIZE_OPTIONS = def(9_122_0_00);
+    public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO_RERANK_ADDED = def(9_123_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,