Merge branch 'main' into downsampling-runs-only-search-nodes

Author: gmarouli

.buildkite/hooks/pre-command

@@ -64,6 +64,14 @@ if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   unset data
 fi
 
+if [[ "${USE_MAVEN_GPG:-}" == "true" ]]; then
+  vault_path="kv/ci-shared/release-eng/team-release-secrets/es-delivery/gpg"
+  ORG_GRADLE_PROJECT_signingKey=$(vault kv get --field="private_key" $vault_path)
+  ORG_GRADLE_PROJECT_signingPassword=$(vault kv get --field="passphase" $vault_path)
+  export ORG_GRADLE_PROJECT_signingKey
+  export ORG_GRADLE_PROJECT_signingPassword
+fi
+
 if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
   DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
   export DRA_VAULT_ROLE_ID_SECRET

.buildkite/pipelines/dra-workflow.yml

@@ -2,6 +2,7 @@ steps:
   - command: .buildkite/scripts/dra-workflow.sh
     env:
       USE_DRA_CREDENTIALS: "true"
+      USE_MAVEN_GPG: "true"
       USE_PROD_DOCKER_CREDENTIALS: "true"
     agents:
       provider: gcp

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -10,12 +10,11 @@
 package org.elasticsearch.gradle.internal.conventions;
 
 import groovy.util.Node;
+import nmcp.NmcpPlugin;
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
 
-import nmcp.NmcpPlugin;
-
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
 import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin;
 import org.elasticsearch.gradle.internal.conventions.util.Util;
@@ -41,6 +40,8 @@
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.initialization.layout.BuildLayout;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
+import org.gradle.plugins.signing.SigningExtension;
+import org.gradle.plugins.signing.SigningPlugin;
 import org.w3c.dom.Element;
 
 import java.io.File;
@@ -69,6 +70,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(PomValidationPrecommitPlugin.class);
         project.getPluginManager().apply(LicensingPlugin.class);
         project.getPluginManager().apply(NmcpPlugin.class);
+        project.getPluginManager().apply(SigningPlugin.class);
         configureJavadocJar(project);
         configureSourcesJar(project);
         configurePomGeneration(project);
@@ -79,6 +81,13 @@ public void apply(Project project) {
     private void configurePublications(Project project) {
         var publishingExtension = project.getExtensions().getByType(PublishingExtension.class);
         var publication = publishingExtension.getPublications().create("elastic", MavenPublication.class);
+        Provider<String> signingKey = project.getProviders().gradleProperty("signingKey");
+        if (signingKey.isPresent()) {
+            SigningExtension signing = project.getExtensions().getByType(SigningExtension.class);
+            signing.useInMemoryPgpKeys(signingKey.get(), project.getProviders().gradleProperty("signingPassword").get());
+            signing.sign(publication);
+        }
+
         project.afterEvaluate(project1 -> {
             if (project1.getPlugins().hasPlugin(ShadowPlugin.class)) {
                 configureWithShadowPlugin(project1, publication);

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -229,6 +229,36 @@ A sample query can look like below:
 
 Note that nested `knn` only supports `score_mode=max`.
 
+## Knn query on a semantic_text field [knn-query-with-semantic-text]
+
+Elasticsearch supports knn queries over a [
+`semantic_text` field](/reference/elasticsearch/mapping-reference/semantic-text.md).
+
+Here is an example using the `query_vector_builder`:
+
+```json
+{
+  "query": {
+    "knn": {
+      "field": "inference_field",
+      "k": 10,
+      "num_candidates": 100,
+      "query_vector_builder": {
+        "text_embedding": {
+          "model_text": "test"
+        }
+      }
+    }
+  }
+}
+```
+
+Note that for `semantic_text` fields, the `model_id` does not have to be
+provided as it can be inferred from the `semantic_text` field mapping.
+
+Knn search using query vectors over `semantic_text` fields is also supported,
+with no change to the API.
+
 ## Knn query with aggregations [knn-query-aggregations]
 
 `knn` query calculates aggregations on top `k` documents from each shard. Thus, the final results from aggregations contain `k * number_of_shards` documents. This is different from the [top level knn section](docs-content://solutions/search/vector/knn.md) where aggregations are calculated on the global top `k` nearest documents.

docs/reference/search-connectors/es-connectors-mongodb.md

@@ -249,8 +249,23 @@ The full host in this example will look like this:
 
 A bug introduced in **8.12.0** causes the Connectors docker image to error out if run using MongoDB as its source. The command line will output the error `cannot import name 'coroutine' from 'asyncio'`. *** This issue is fixed in versions *8.12.2** and **8.13.0**. ** This bug does not affect Elastic managed connectors.
 
-See [Known issues](/release-notes/known-issues.md) for any issues affecting all connectors.
+#### UUIDs are not correctly deserialized, causing problems with ingesting documents into Elasticsearch
+
+MongoDB has special handling of UUID type: there is a legacy and a modern approach. You can read [official docs](https://pymongo.readthedocs.io/en/stable/examples/uuid.html) about the details.
+
+With connector framework version 9.0.3, we improved how standard UUIDs are handled. Now, the MongoDB connector can correctly deserialize UUIDs into valid Elasticsearch values. However, for legacy UUIDs or older connector versions, you might need to adjust the connection string to specify the UUID representation.
+
+For example, if you are using the modern UUID representation, adding the `uuidRepresentation=standard` query parameter to the MongoDB connection URI in the `host` Rich Configurable Field will allow the connector to properly handle UUIDs. With this change, the full `host` Rich Configurable Field value could look like this:`mongodb+srv://my_username:[email protected]/mydb?w=majority&uuidRepresentation=standard`
 
+If you’re using a legacy UUID representation, you should adjust the connection URI accordingly. For example:
+
+- C#: `uuidRepresentation=csharpLegacy`
+- Java: `uuidRepresentation=javaLegacy`
+- Python: `uuidRepresentation=pythonLegacy`
+
+You can find a full explanation in the [official docs](https://pymongo.readthedocs.io/en/stable/examples/uuid.html#configuring-a-uuid-representation).
+
+See [Known issues](/release-notes/known-issues.md) for any issues affecting all connectors.
 
 ### Troubleshooting [es-connectors-mongodb-client-troubleshooting]
 

docs/reference/search-connectors/es-connectors-sharepoint-online.md

@@ -592,6 +592,19 @@ make ftest NAME=sharepoint_online DATA_SIZE=small
 
         **Workaround**: Disable `Enumerate All Sites?`, and configure full site paths for all desired sites.
 
+* **ACL is not properly inherited for Site Pages and List Items inside of a folder with Unique Permissions when DLS is enabled with Fetch unique list item permissions, Fetch unique page permissions or Fetch drive item permissions**
+
+    There is a known issue with ACL propagation when List Items, Site Pages or Drive Items are located inside of a folder that has Unique permissions enabled. Consider the following example:
+
+    ```
+    [0] Root Site (Access: All)
+    [1]  Subsite Travel (Access: inherit)
+    [2]    Folder "/es" (Access: Spanish Employees)
+    [3]      Page "destinations.html" (Access: inherit)
+    ```
+
+    Expected permissions for `destinations.html` should be `Access: Spanish Employees`, but will be `Access: All`, because permissions will be assumed from Subsite Travel, rather than folder "/es".
+
 
 Refer to [Known issues](/release-notes/known-issues.md) for a list of known issues for all connectors.
 

modules/data-streams/build.gradle

@@ -30,6 +30,10 @@ tasks.withType(StandaloneRestIntegTestTask).configureEach {
   usesDefaultDistribution("to be triaged")
 }
 
+tasks.named("internalClusterTest").configure {
+  systemProperty 'es.logs_stream_feature_flag_enabled', 'true'
+}
+
 if (buildParams.inFipsJvm){
   // These fail in CI but only when run as part of checkPart2 and not individually.
   // Tracked in :