Merge branch 'main' into esql_danger_zone_2

Author: nik9000

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -34,6 +34,7 @@
 import java.io.File;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Stream;
 
 import javax.inject.Inject;
@@ -50,6 +51,8 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
 
     public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
 
+    public static final Set<String> TEST_TASKS_WITH_ENTITLEMENTS = Set.of("test", "internalClusterTest");
+
     @Inject
     protected abstract ProviderFactory getProviderFactory();
 
@@ -174,14 +177,23 @@ public void execute(Task t) {
             nonInputProperties.systemProperty("workspace.dir", Util.locateElasticsearchWorkspace(project.getGradle()));
             // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
             nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
+            if (test.getName().equals("internalClusterTest")) {
+                // configure a node home directory independent of the Java temp dir so that entitlements can be properly enforced
+                nonInputProperties.systemProperty("tempDir", test.getWorkingDir().toPath().resolve("nodesTemp"));
+            }
 
             SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
             SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
             SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
-            if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
+            SourceSet internalClusterTestSourceSet = sourceSets.findByName("internalClusterTest");
+
+            if (TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) && mainSourceSet != null && testSourceSet != null) {
                 FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
                 FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
-                FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
+                FileCollection internalClusterTestRuntime = ("internalClusterTest".equals(test.getName())
+                    && internalClusterTestSourceSet != null) ? internalClusterTestSourceSet.getRuntimeClasspath() : project.files();
+                FileCollection testOnlyFiles = testRuntime.plus(internalClusterTestRuntime).minus(mainRuntime);
+
                 test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
             }
 
@@ -241,14 +253,15 @@ public void execute(Task t) {
      * Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
      */
     private void configureJavaBaseModuleOptions(Project project) {
-        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
-            FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            // patch immutable collections only for "test" task
+            FileCollection patchedImmutableCollections = test.getName().equals("test") ? patchedImmutableCollections(project) : null;
             if (patchedImmutableCollections != null) {
                 test.getInputs().files(patchedImmutableCollections);
                 test.systemProperty("tests.hackImmutableCollections", "true");
             }
 
-            FileCollection entitlementBridge = entitlementBridge(project);
+            FileCollection entitlementBridge = TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) ? entitlementBridge(project) : null;
             if (entitlementBridge != null) {
                 test.getInputs().files(entitlementBridge);
             }
@@ -312,27 +325,30 @@ private static void configureEntitlements(Project project) {
         }
         FileCollection bridgeFiles = bridgeConfig;
 
-        project.getTasks().withType(Test.class).configureEach(test -> {
-            // See also SystemJvmOptions.maybeAttachEntitlementAgent.
-
-            // Agent
-            if (agentFiles.isEmpty() == false) {
-                test.getInputs().files(agentFiles);
-                test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
-                test.systemProperty("jdk.attach.allowAttachSelf", true);
-            }
+        project.getTasks()
+            .withType(Test.class)
+            .matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
+            .configureEach(test -> {
+                // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+
+                // Agent
+                if (agentFiles.isEmpty() == false) {
+                    test.getInputs().files(agentFiles);
+                    test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
+                    test.systemProperty("jdk.attach.allowAttachSelf", true);
+                }
 
-            // Bridge
-            if (bridgeFiles.isEmpty() == false) {
-                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
-                test.getInputs().files(bridgeFiles);
-                // Tests may not be modular, but the JDK still is
-                test.jvmArgs(
-                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
-                        + modulesContainingEntitlementInstrumentation
-                );
-            }
-        });
+                // Bridge
+                if (bridgeFiles.isEmpty() == false) {
+                    String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                    test.getInputs().files(bridgeFiles);
+                    // Tests may not be modular, but the JDK still is
+                    test.jvmArgs(
+                        "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                            + modulesContainingEntitlementInstrumentation
+                    );
+                }
+            });
     }
 
 }

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -58,9 +58,12 @@ public void apply(Project project) {
         });
 
         if (project.getRootProject().getName().equals("elasticsearch")) {
-            project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
-                test.systemProperty("es.entitlement.enableForTests", "true");
-            });
+            project.getTasks()
+                .withType(Test.class)
+                .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
+                .configureEach(test -> {
+                    test.systemProperty("es.entitlement.enableForTests", "true");
+                });
         }
     }
 }

docs/changelog/121914.yaml

@@ -0,0 +1,6 @@
+pr: 129282
+summary: "Fix query rewrite logic to preserve `boosts` and `queryName` for `match`,\
+  \ `knn`, and `sparse_vector` queries on semantic_text fields"
+area: Search
+type: bug
+issues: []

docs/changelog/129282.yaml

@@ -0,0 +1,5 @@
+pr: 129848
+summary: "[ML] Add Azure AI Rerank support to the Inference Plugin"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/129848.yaml

@@ -0,0 +1,6 @@
+pr: 130940
+summary: Block trained model updates from inference
+area: Machine Learning
+type: enhancement
+issues:
+ - 129999

docs/changelog/130940.yaml

@@ -0,0 +1,5 @@
+pr: 131296
+summary: Enable failure store for newly created APM datastreams
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/131296.yaml

@@ -0,0 +1,6 @@
+pr: 131391
+summary: Fix bug in point in time response
+area: Search
+type: bug
+issues:
+ - 131026

docs/changelog/131391.yaml

@@ -50,7 +50,7 @@ configure(childProjects.values()) {
   // Omit oddball libraries that aren't in server.
   def nonServerLibs = ['plugin-scanner']
   if (false == nonServerLibs.contains(project.name)) {
-    project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
+    project.getTasks().withType(Test.class).matching(test -> ['test', 'internalClusterTest'].contains(test.name)).configureEach(test -> {
       test.systemProperty('es.entitlement.enableForTests', 'true')
     })
   }

libs/build.gradle

@@ -74,150 +74,3 @@ teardown:
   - match: { _source.bytes_source_field: "1kb" }
   - match: { _source.conditional_field: "bar" }
   - is_false: _source.bytes_target_field
-
----
-"Test conditional processor with fields API":
-  - do:
-      ingest.put_pipeline:
-        id: "my_pipeline"
-        body:
-          description: "_description"
-          processors:
-            - set:
-                if: "field('get.field').get('') == 'one'"
-                field: "one"
-                value: 1
-            - set:
-                if: "field('get.field').get('') == 'two'"
-                field: "missing"
-                value: "missing"
-            - set:
-                if: " /* avoid yaml stash */ $('get.field', 'one') == 'one'"
-                field: "dollar"
-                value: true
-            - set:
-                if: "field('missing.field').get('fallback') == 'fallback'"
-                field: "fallback"
-                value: "fallback"
-            - set:
-                if: "field('nested.array.get.with.index.field').get(1, null) == 'two'"
-                field: "two"
-                value: 2
-            - set:
-                if: "field('getName.field').getName() == 'getName.field'"
-                field: "three"
-                value: 3
-            - set:
-                if: "field('existing.field').exists()"
-                field: "four"
-                value: 4
-            - set:
-                if: "!field('empty.field').isEmpty()"
-                field: "missing"
-                value: "missing"
-            - set:
-                if: "field('size.field').size() == 2"
-                field: "five"
-                value: 5
-            - set:
-                if: >
-                  def iterator = field('iterator.field').iterator();
-                  def sum = 0;
-                  while (iterator.hasNext()) {
-                    sum += iterator.next();
-                  }
-                  return sum == 6;
-                field: "six"
-                value: 6
-            - set:
-                if: "field('hasValue.field').hasValue(v -> v == 'two')"
-                field: "seven"
-                value: 7
-  - match: { acknowledged: true }
-
-  - do:
-      index:
-        index: test
-        id: "1"
-        pipeline: "my_pipeline"
-        body:
-          get.field: "one"
-          nested:
-            array:
-              get.with.index.field: ["one", "two", "three"]
-          getName.field: "my_name"
-          existing.field: "indeed"
-          empty.field: []
-          size.field: ["one", "two"]
-          iterator.field: [1, 2, 3]
-          hasValue.field: ["one", "two", "three"]
-
-  - do:
-      get:
-        index: test
-        id: "1"
-  - match: { _source.get\.field: "one" }
-  - match: { _source.one: 1 }
-  - is_false: _source.missing
-  - is_true: _source.dollar
-  - match: { _source.fallback: "fallback" }
-  - match: { _source.nested.array.get\.with\.index\.field: ["one", "two", "three"] }
-  - match: { _source.two: 2 }
-  - match: { _source.three: 3 }
-  - match: { _source.four: 4 }
-  - match: { _source.five: 5 }
-  - match: { _source.six: 6 }
-  - match: { _source.seven: 7 }
-
----
-"Test fields iterator is unmodifiable":
-  - do:
-      ingest.put_pipeline:
-        id: "my_pipeline"
-        body:
-          description: "_description"
-          processors:
-            - set:
-                if: >
-                  def iterator = field('iterator.field').iterator();
-                  def sum = 0;
-                    while (iterator.hasNext()) {
-                      sum += iterator.next();
-                      iterator.remove();
-                    }
-                  return sum == 6;
-                field: "sum"
-                value: 6
-  - match: { acknowledged: true }
-
-  - do:
-      index:
-        index: test
-        id: "1"
-        pipeline: "my_pipeline"
-        body:
-          test.field: [1, 2, 3]
-  - match: { error: null }
-
-  - do:
-      index:
-        index: test
-        id: "2"
-        pipeline: "my_pipeline"
-        body:
-          iterator.field: [1, 2, 3]
-      catch: bad_request
-  - length: { error.root_cause: 1 }
-
-  - do:
-      get:
-        index: test
-        id: "1"
-  - match: { _source.test\.field: [1, 2, 3] }
-  - is_false: _source.sum
-
-  - do:
-      get:
-        index: test
-        id: "2"
-      catch: missing