Merge branch 'main' into cs-embedding-types

Author: jonathan-buttner

.buildkite/hooks/pre-command

@@ -64,6 +64,14 @@ if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   unset data
 fi
 
+if [[ "${USE_MAVEN_GPG:-}" == "true" ]]; then
+  vault_path="kv/ci-shared/release-eng/team-release-secrets/es-delivery/gpg"
+  ORG_GRADLE_PROJECT_signingKey=$(vault kv get --field="private_key" $vault_path)
+  ORG_GRADLE_PROJECT_signingPassword=$(vault kv get --field="passphase" $vault_path)
+  export ORG_GRADLE_PROJECT_signingKey
+  export ORG_GRADLE_PROJECT_signingPassword
+fi
+
 if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
   DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
   export DRA_VAULT_ROLE_ID_SECRET

.buildkite/pipelines/dra-workflow.yml

@@ -2,6 +2,7 @@ steps:
   - command: .buildkite/scripts/dra-workflow.sh
     env:
       USE_DRA_CREDENTIALS: "true"
+      USE_MAVEN_GPG: "true"
       USE_PROD_DOCKER_CREDENTIALS: "true"
     agents:
       provider: gcp

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -24,7 +24,7 @@ select(.active_release == true) |
 (.build_candidates | to_entries | sort_by(.value.completed_at))) |
 last | .value.manifest_url")"
 
-if [[ -z "$MANIFEST_URL" ]]; then
+if [[ -z "$MANIFEST_URL" ]] || [[ "$MANIFEST_URL" == "null" ]]; then
    echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]."
    echo "Skipping BC upgrade tests."
    exit 0

.buildkite/scripts/run-pr-upgrade-tests.sh

@@ -18,7 +18,8 @@ fi
 
 # Identify the merge base of the current commit (branch) and the base branch of the pull request.
 # PR upgrade tests are run from the merge base to the current commit.
-BASE_COMMIT=$(git merge-base $BUILDKITE_PULL_REQUEST_BASE_BRANCH $BUILDKITE_COMMIT)
+git fetch origin $BUILDKITE_PULL_REQUEST_BASE_BRANCH
+BASE_COMMIT=$(git merge-base origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH $BUILDKITE_COMMIT)
 
 VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
 

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -10,12 +10,11 @@
 package org.elasticsearch.gradle.internal.conventions;
 
 import groovy.util.Node;
+import nmcp.NmcpPlugin;
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
 
-import nmcp.NmcpPlugin;
-
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
 import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin;
 import org.elasticsearch.gradle.internal.conventions.util.Util;
@@ -41,6 +40,8 @@
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.initialization.layout.BuildLayout;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
+import org.gradle.plugins.signing.SigningExtension;
+import org.gradle.plugins.signing.SigningPlugin;
 import org.w3c.dom.Element;
 
 import java.io.File;
@@ -69,6 +70,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(PomValidationPrecommitPlugin.class);
         project.getPluginManager().apply(LicensingPlugin.class);
         project.getPluginManager().apply(NmcpPlugin.class);
+        project.getPluginManager().apply(SigningPlugin.class);
         configureJavadocJar(project);
         configureSourcesJar(project);
         configurePomGeneration(project);
@@ -79,6 +81,13 @@ public void apply(Project project) {
     private void configurePublications(Project project) {
         var publishingExtension = project.getExtensions().getByType(PublishingExtension.class);
         var publication = publishingExtension.getPublications().create("elastic", MavenPublication.class);
+        Provider<String> signingKey = project.getProviders().gradleProperty("signingKey");
+        if (signingKey.isPresent()) {
+            SigningExtension signing = project.getExtensions().getByType(SigningExtension.class);
+            signing.useInMemoryPgpKeys(signingKey.get(), project.getProviders().gradleProperty("signingPassword").get());
+            signing.sign(publication);
+        }
+
         project.afterEvaluate(project1 -> {
             if (project1.getPlugins().hasPlugin(ShadowPlugin.class)) {
                 configureWithShadowPlugin(project1, publication);

docs/changelog/125921.yaml

@@ -0,0 +1,18 @@
+pr: 125921
+summary: Allow direct IO for BBQ rescoring
+area: Vector Search
+type: feature
+highlight:
+  title: Allow direct IO for BBQ rescoring
+  body: |-
+    BBQ rescoring performance can be drastically affected by the amount of available
+    off-heap RAM for use by the system page cache. When there is not enough off-heap RAM
+    to fit all the vector data in memory, BBQ search latencies can be affected by as much as 5000x.
+    Specifying the `vector.rescoring.directio=true` Java option on all vector search
+    nodes modifies rescoring to use direct IO, which eliminates these very high latencies
+    from searches in low-memory scenarios, at a cost of a reduction
+    in vector search performance for BBQ indices when the vectors do all fit in memory.
+
+    This option is released in 9.1 as a tech preview whilst we analyse its effect
+    for a variety of use cases.
+issues: []

docs/changelog/129990.yaml

@@ -0,0 +1,5 @@
+pr: 129990
+summary: Make forecast write load accurate when shard numbers change
+area: Allocation
+type: bug
+issues: []

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -229,6 +229,39 @@ A sample query can look like below:
 
 Note that nested `knn` only supports `score_mode=max`.
 
+## Knn query on a semantic_text field [knn-query-with-semantic-text]
+
+Elasticsearch supports knn queries over a [
+`semantic_text` field](/reference/elasticsearch/mapping-reference/semantic-text.md).
+
+Here is an example using the `query_vector_builder`:
+
+```json
+{
+  "query": {
+    "knn": {
+      "field": "inference_field",
+      "k": 10,
+      "num_candidates": 100,
+      "query_vector_builder": {
+        "text_embedding": {
+          "model_text": "test"
+        }
+      }
+    }
+  },
+  "_source": {
+    "exclude": "inference_field.inference.chunks"
+  }
+}
+```
+
+Note that for `semantic_text` fields, the `model_id` does not have to be
+provided as it can be inferred from the `semantic_text` field mapping.
+
+Knn search using query vectors over `semantic_text` fields is also supported,
+with no change to the API.
+
 ## Knn query with aggregations [knn-query-aggregations]
 
 `knn` query calculates aggregations on top `k` documents from each shard. Thus, the final results from aggregations contain `k * number_of_shards` documents. This is different from the [top level knn section](docs-content://solutions/search/vector/knn.md) where aggregations are calculated on the global top `k` nearest documents.

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java

@@ -222,9 +222,7 @@ private void updateSettingsOnIndices(
         Map<String, Object> settingsToApply = new HashMap<>();
         List<String> appliedToDataStreamOnly = new ArrayList<>();
         List<String> appliedToDataStreamAndBackingIndices = new ArrayList<>();
-        Settings effectiveSettings = dataStream.getEffectiveSettings(
-            clusterService.state().projectState(projectResolver.getProjectId()).metadata()
-        );
+        Settings effectiveSettings = dataStream.getEffectiveSettings(projectResolver.getProjectMetadata(clusterService.state()));
         for (String settingName : requestSettings.keySet()) {
             if (APPLY_TO_BACKING_INDICES.contains(settingName)) {
                 settingsToApply.put(settingName, effectiveSettings.get(settingName));

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java

@@ -151,7 +151,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @FixForMultiProject(description = "Replace ProjectId.DEFAULT")
     static EnterpriseGeoIpTaskState getEnterpriseGeoIpTaskState(ClusterState state) {
         PersistentTasksCustomMetadata.PersistentTask<?> task = getTaskWithId(
-            state.projectState(ProjectId.DEFAULT).metadata(),
+            state.metadata().getProject(ProjectId.DEFAULT),
             EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER
         );
         return (task == null) ? null : (EnterpriseGeoIpTaskState) task.getState();