Merge branch 'main' into feature/knn-vector-rescore-query-docs

Author: carlosdelest

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -24,7 +24,7 @@ abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
         remoteGitRepo = new File(setupGitRemote(), '.git')
-        "git clone ${remoteGitRepo.absolutePath} cloned".execute(Collections.emptyList(), testProjectDir.root).waitFor()
+        execute("git clone ${remoteGitRepo.absolutePath} cloned", testProjectDir.root)
         buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
         settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.jdk.RuntimeVersionFeature;
 
@@ -26,7 +27,9 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-        boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
+        boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
         return Stream.of(
             Stream.of(
                 /*

docs/changelog/119780.yaml

@@ -0,0 +1,5 @@
+pr: 119780
+summary: Add index and reindex request settings to speed up reindex
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/119798.yaml

@@ -0,0 +1,5 @@
+pr: 119798
+summary: "Add a `PostAnalysisAware,` distribute verification"
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/connector/docs/connectors-sharepoint-online.asciidoc

@@ -75,12 +75,10 @@ Follow these steps:
 * Leave the *Redirect URIs* blank for now.
 * *Register* the application.
 * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy.
-* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**.
-* Select **New client secret**
-* Pick a name for your client secret.
-Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.)
-** Save the client secret **Secret ID** before leaving this screen.
-** Save the client secret **Value** before leaving this screen.
+* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place
+* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**.
+* Select **Upload certificate**
+* Upload the certificate created in one of previous steps: `azure_app.crt`
 * Set up the permissions the OAuth App will request from the Azure Portal service account.
 ** Navigate to **API Permissions** and click **Add Permission**.
 ** Add **application permissions** until the list looks like the following:
@@ -114,6 +112,24 @@ When entities are not available via the Graph API the connector falls back to us
 [discrete#es-connectors-sharepoint-online-oauth-app-permissions]
 ====== SharePoint permissions
 
+Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration:
+
+* *Tenants created after November 1st, 2024*: Certificate authentication is required
+* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026
+
+[discrete#es-connectors-sharepoint-online-oauth-app-certificate-auth]
+===== Certificate Authentication
+
+This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App.
+
+[discrete#es-connectors-sharepoint-online-oauth-app-secret-auth]
+===== Secret Authentication
+
+[IMPORTANT]
+====
+This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026.
+====
+
 Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^].
 
 * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false`
@@ -219,8 +235,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance.
 Client ID::
 The client id to authenticate with SharePoint Online.
 
+Authentication Method::
+Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended.
+
 Secret value::
-The secret value to authenticate with SharePoint Online.
+The secret value to authenticate with SharePoint Online, if Authentication Method: `secret` is chosen.
+
+Content of certificate file::
+Content of certificate file if Authentication Method: `certificate` is chosen.
+
+Content of private key file::
+Content of private key file if Authentication Method: `certificate` is chosen.
 
 Comma-separated list of sites::
 List of site collection names or paths to fetch from SharePoint.
@@ -588,12 +613,10 @@ Follow these steps:
 * Leave the *Redirect URIs* blank for now.
 * *Register* the application.
 * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy.
-* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**.
-* Select **New client secret**
-* Pick a name for your client secret.
-Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.)
-** Save the client secret **Secret ID** before leaving this screen.
-** Save the client secret **Value** before leaving this screen.
+* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place
+* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**.
+* Select **Upload certificate**
+* Upload the certificate created in one of previous steps: `azure_app.crt`
 * Set up the permissions the OAuth App will request from the Azure Portal service account.
 ** Navigate to **API Permissions** and click **Add Permission**.
 ** Add **application permissions** until the list looks like the following:
@@ -627,6 +650,23 @@ When entities are not available via the Graph API the connector falls back to us
 [discrete#es-connectors-sharepoint-online-client-oauth-app-permissions]
 ====== SharePoint permissions
 
+Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration:
+* *Tenants created after November 1st, 2024*: Certificate authentication is required
+* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026
+
+[discrete#es-connectors-sharepoint-online-client-oauth-app-certificate-auth]
+===== Certificate Authentication
+
+This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App.
+
+[discrete#es-connectors-sharepoint-online-client-oauth-app-secret-auth]
+===== Secret Authentication
+
+[IMPORTANT]
+====
+This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026.
+====
+
 Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^].
 
 * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false`
@@ -742,8 +782,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance.
 `client_id`::
 The client id to authenticate with SharePoint Online.
 
+`auth_method`::
+Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended.
+
 `secret_value`::
-The secret value to authenticate with SharePoint Online.
+The secret value to authenticate with SharePoint Online, if auth_method: `secret` is chosen.
+
+`certificate`::
+Content of certificate file if auth_method: `certificate` is chosen.
+
+`private_key`::
+Content of private key file if auth_method: `certificate` is chosen.
 
 `site_collections`::
 List of site collection names or paths to fetch from SharePoint.

docs/reference/query-dsl/intervals-query.asciidoc

@@ -303,12 +303,22 @@ matches. Defaults to `-1`.
 
 If unspecified or set to `-1`, there is no width restriction on the match. If
 set to `0`, the terms must appear next to each other.
+
+Internal intervals can have their own `max_gaps` values. In this case
+we first find internal intervals with their `max_gaps` values, and then
+combine them to see if a gap between internal intervals match
+the value of `max_gaps` of the `all_of` rule.
+
+For examples, how `max_gaps` works, see <<interval-max_gaps-all-rule>>.
 --
 
 `ordered`::
 (Optional, Boolean) If `true`, intervals produced by the rules should appear in
 the order in which they are specified. Defaults to `false`.
 
+If `ordered` is `false`, intervals can appear in any order,
+including overlapping with each other.
+
 `filter`::
 (Optional, <<interval_filter,interval filter>> rule object) Rule used to filter
 returned intervals.
@@ -468,3 +478,94 @@ This query does *not* match a document containing the phrase `hot porridge is
 salty porridge`, because the intervals returned by the match query for `hot
 porridge` only cover the initial two terms in this document, and these do not
 overlap the intervals covering `salty`.
+
+[[interval-max_gaps-all-rule]]
+===== max_gaps in `all_of` ordered and unordered rule
+
+The following `intervals` search returns documents containing `my
+favorite food` without any gap, followed by `cold porridge` that
+can have at most 4 tokens between "cold" and "porridge". These
+two inner intervals when combined in the outer `all_of` interval,
+must have at most 1 gap between each other.
+
+Because the `all_of` rule has `ordered` set to `true`, the inner
+intervals are expected to be in the provided order. Thus,
+this search would match a `my_text` value of `my favorite food is cold
+porridge` but not `when it's cold my favorite food is porridge`.
+
+[source,console]
+--------------------------------------------------
+POST _search
+{
+  "query": {
+    "intervals" : {
+      "my_text" : {
+        "all_of" : {
+          "ordered" : true,     <1>
+          "max_gaps": 1,
+          "intervals" : [
+            {
+              "match" : {
+                "query" : "my favorite food",
+                "max_gaps" : 0,
+                "ordered" : true
+              }
+            },
+            {
+              "match" : {
+                "query" : "cold porridge",
+                "max_gaps" : 4,
+                "ordered" : true
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The `ordered` parameter is set to `true`, so intervals must appear in the order specified.
+
+
+Below is the same query, but with `ordered` set to `false`. This means that
+intervals can appear in any order, even overlap with each other.
+Thus, this search would match a `my_text` value of `my favorite food is cold
+porridge`, as well as `when it's cold my favorite food is porridge`.
+In `when it's cold my favorite food is porridge`, `cold .... porridge` interval
+overlaps with `my favorite food` interval.
+
+[source,console]
+--------------------------------------------------
+POST _search
+{
+  "query": {
+    "intervals" : {
+      "my_text" : {
+        "all_of" : {
+          "ordered" : false, <1>
+          "max_gaps": 1,
+          "intervals" : [
+            {
+              "match" : {
+                "query" : "my favorite food",
+                "max_gaps" : 0,
+                "ordered" : true
+              }
+            },
+            {
+              "match" : {
+                "query" : "cold porridge",
+                "max_gaps" : 4,
+                "ordered" : true
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The `ordered` parameter is set to `true`, so intervals can appear in any order,
+even overlap with each other.

docs/reference/rest-api/security/get-service-accounts.asciidoc

@@ -276,6 +276,21 @@ GET /_security/service/elastic/fleet-server
             "view_index_metadata"
           ],
           "allow_restricted_indices": false
+        },
+        {
+          "names": [
+            "agentless-*",
+          ],
+          "privileges": [
+            "read",
+            "write",
+            "monitor",
+            "create_index",
+            "auto_configure",
+            "maintenance",
+            "view_index_metadata"
+          ],
+          "allow_restricted_indices": false
         }
       ],
       "applications": [

docs/reference/scripting/using.asciidoc

@@ -201,8 +201,13 @@ when you're creating <<runtime-mapping-fields,runtime fields>>.
 [[script-stored-scripts]]
 === Store and retrieve scripts
 You can store and retrieve scripts from the cluster state using the
-<<stored-script-apis,stored script APIs>>. Stored scripts reduce compilation
-time and make searches faster.
+<<stored-script-apis,stored script APIs>>. Stored scripts allow you to reference 
+shared scripts for operations like scoring, aggregating, filtering, and 
+reindexing. Instead of embedding scripts inline in each query, you can reference 
+these shared operations.
+
+Stored scripts can also reduce request payload size. Depending on script size 
+and request frequency, this can help lower latency and data transfer costs.
 
 NOTE: Unlike regular scripts, stored scripts require that you specify a script
 language using the `lang` parameter.