Merge branch 'main' into limit_concurrent_node_requests

Author: idegtiarenko

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -133,7 +133,7 @@ develocity {
         }
       } else {
         tag 'LOCAL'
-        if (providers.systemProperty('idea.active').present) {
+        if (System.getProperty('idea.active') == 'true') {
           tag 'IDEA'
         }
       }

docs/README.md

@@ -17,7 +17,7 @@ Docs live in **three places**:
 
 1. **Reference content** lives in this repo. This covers low-level stuff like settings and configuration information that is tightly coupled to code.
 - 👩🏽‍💻 **Engineers** own the bulk of this content.
-2.  **API reference docs** live in the [Elasticsearch specification](https://github.com/elastic/elasticsearch-specification)
+2.  **API reference docs** live in the [Elasticsearch specification](https://github.com/elastic/elasticsearch-specification/blob/main/README.md#how-to-generate-the-openapi-representation)
     - This is where you need to update API docs published in the [new API docs system](https://www.elastic.co/docs/api/doc/elasticsearch/v8/)
 - 👩🏽‍💻 **Engineers** own this content.
 3. **Narrative, overview, and conceptual content** mostly lives in the [`docs-content`](https://github.com/elastic/docs-content/) repo. 
@@ -106,4 +106,4 @@ This allows slightly more expressive testing of the snippets. Since that syntax
 % TEST[s/\n$/\nstartyaml\n  - compare_analyzers: {index: thai_example, first: thai, second: rebuilt_thai}\nendyaml\n/]
 ```
 
-Any place you can use json you can use elements like `$body.path.to.thing` which is replaced on the fly with the contents of the thing at `path.to.thing` in the last response.
+Any place you can use json you can use elements like `$body.path.to.thing` which is replaced on the fly with the contents of the thing at `path.to.thing` in the last response.

docs/changelog/122615.yaml

@@ -0,0 +1,8 @@
+pr: 122615
+summary: Enable synthetic recovery source by default when synthetic source is enabled.
+  Using synthetic recovery source significantly improves indexing performance compared
+  to regular recovery source.
+area: Mapping
+type: enhancement
+issues:
+ - 116726

docs/reference/ingestion-tools/search-connectors/es-connectors-servicenow.md

@@ -205,9 +205,12 @@ The ServiceNow connector supports roles for access control lists (ACLs) to enabl
 
 For services other than these defaults, the connector iterates over access controls with `read` operations and finds the respective roles for those services.
 
+:::{important}
+The ServiceNow connector applies access control at the service (table) level. This means documents within a given ServiceNow table share the same access control settings. Users with permission to a table can access all documents from that table in Elasticsearch.
+:::
+
 ::::{note}
 The ServiceNow connector does not support scripted and conditional permissions.
-
 ::::
 
 

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureTests.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.index.mapper.extras;
 
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesBuilder;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.plugins.Plugin;
@@ -19,7 +20,6 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -59,10 +59,7 @@ public void testRankFeatureInIndex() {
         Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
         assertEquals(1, fooRankField.size());
         assertThat(fooRankField, Matchers.hasKey("rank_feature"));
-        assertEquals(
-            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
-            fooRankField.get("rank_feature")
-        );
+        assertEquals(fieldCapabilities("fooRank"), fooRankField.get("rank_feature"));
     }
 
     public void testRankFeatureInIndexAfterRestart() throws Exception {
@@ -79,10 +76,7 @@ public void testRankFeatureInIndexAfterRestart() throws Exception {
         Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
         assertEquals(1, fooRankField.size());
         assertThat(fooRankField, Matchers.hasKey("rank_feature"));
-        assertEquals(
-            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
-            fooRankField.get("rank_feature")
-        );
+        assertEquals(fieldCapabilities("fooRank"), fooRankField.get("rank_feature"));
     }
 
     public void testAllRankFeatureReturnedIfOneIsPresent() {
@@ -98,18 +92,16 @@ public void testAllRankFeatureReturnedIfOneIsPresent() {
         Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
         assertEquals(1, fooRankField.size());
         assertThat(fooRankField, Matchers.hasKey("rank_feature"));
-        assertEquals(
-            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
-            fooRankField.get("rank_feature")
-        );
+        assertEquals(fieldCapabilities("fooRank"), fooRankField.get("rank_feature"));
         assertThat(response.get(), Matchers.hasKey("barRank"));
         // Check the capabilities for the 'barRank' field.
         Map<String, FieldCapabilities> barRankField = response.getField("barRank");
         assertEquals(1, barRankField.size());
         assertThat(barRankField, Matchers.hasKey("rank_feature"));
-        assertEquals(
-            new FieldCapabilities("barRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
-            barRankField.get("rank_feature")
-        );
+        assertEquals(fieldCapabilities("barRank"), barRankField.get("rank_feature"));
+    }
+
+    private static FieldCapabilities fieldCapabilities(String fieldName) {
+        return new FieldCapabilitiesBuilder(fieldName, "rank_feature").isAggregatable(false).build();
     }
 }

muted-tests.yml

@@ -130,9 +130,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/118914
 - class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectoryRunAsIT
   issue: https://github.com/elastic/elasticsearch/issues/115727
-- class: org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapperTests
-  method: testCartesianBoundsBlockLoader
-  issue: https://github.com/elastic/elasticsearch/issues/119201
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test start/stop/start transform}
   issue: https://github.com/elastic/elasticsearch/issues/119508
@@ -266,9 +263,6 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_488}
   issue: https://github.com/elastic/elasticsearch/issues/121611
-- class: org.elasticsearch.index.mapper.ShapeGeometryFieldMapperTests
-  method: testCartesianBoundsBlockLoader
-  issue: https://github.com/elastic/elasticsearch/issues/122661
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_408}
   issue: https://github.com/elastic/elasticsearch/issues/122681
@@ -339,6 +333,9 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/3rd_party_deployment/Test start deployment fails while model download in progress}
   issue: https://github.com/elastic/elasticsearch/issues/120814
+- class: org.elasticsearch.test.apmintegration.MetricsApmIT
+  method: testApmIntegration
+  issue: https://github.com/elastic/elasticsearch/issues/124106
 
 # Examples:
 #

plugins/discovery-ec2/build.gradle

@@ -46,13 +46,12 @@ esplugin.bundleSpec.from('config/discovery-ec2') {
 }
 
 tasks.register("writeTestJavaPolicy") {
+  boolean inFips = buildParams.inFipsJvm
+  inputs.property("inFipsJvm", inFips)
+  final File javaPolicy = new File(layout.buildDirectory.asFile.get(), "tmp/java.policy")
+  outputs.file(javaPolicy)
   doLast {
-    final File tmp = file("${buildDir}/tmp")
-    if (tmp.exists() == false && tmp.mkdirs() == false) {
-      throw new GradleException("failed to create temporary directory [${tmp}]")
-    }
-    final File javaPolicy = file("${tmp}/java.policy")
-    if (buildParams.inFipsJvm) {
+    if (inFips) {
       javaPolicy.write(
         [
           "grant {",
@@ -95,9 +94,9 @@ tasks.withType(Test).configureEach {
   // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property
   // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'`
   if (buildParams.inFipsJvm){
-    nonInputProperties.systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy"
+    nonInputProperties.systemProperty 'java.security.policy', "=file://${layout.buildDirectory.asFile.get()}/tmp/java.policy"
   } else {
-    nonInputProperties.systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy"
+    nonInputProperties.systemProperty 'java.security.policy', "file://${layout.buildDirectory.asFile.get()}/tmp/java.policy"
   }
 }
 

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.example;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
@@ -86,14 +87,14 @@ public void authorizeClusterAction(RequestInfo requestInfo, AuthorizationInfo au
     }
 
     @Override
-    public void authorizeIndexAction(
+    SubscribableListener<IndexAuthorizationResult> void authorizeIndexAction(
         RequestInfo requestInfo,
         AuthorizationInfo authorizationInfo,
         AsyncSupplier<ResolvedIndices> indicesAsyncSupplier,
-        ProjectMetadata project,
-        ActionListener<IndexAuthorizationResult> listener
+        ProjectMetadata project
     ) {
         if (isSuperuser(requestInfo.getAuthentication().getEffectiveSubject().getUser())) {
+            ActionListener<IndexAuthorizationResult> listener = new SubscribableListener<>();
             indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
                 Map<String, IndexAccessControl> indexAccessControlMap = new HashMap<>();
                 for (String name : resolvedIndices.getLocal()) {
@@ -103,8 +104,9 @@ public void authorizeIndexAction(
                     new IndicesAccessControl(true, Collections.unmodifiableMap(indexAccessControlMap));
                 listener.onResponse(new IndexAuthorizationResult(indicesAccessControl));
             }, listener::onFailure));
+            return listener;
         } else {
-            listener.onResponse(new IndexAuthorizationResult(IndicesAccessControl.DENIED));
+            return SubscribableListener.succcess(new IndexAuthorizationResult(IndicesAccessControl.DENIED));
         }
     }
 

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -36,7 +36,6 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]"))
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
-        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
         .build();
 

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

@@ -36,7 +36,6 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .module("data-streams")
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
-        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
         .build();