Merge branch 'main' into non-issue/knn-k-param

Author: ioanatia

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -24,7 +24,7 @@ select(.active_release == true) |
 (.build_candidates | to_entries | sort_by(.value.completed_at))) |
 last | .value.manifest_url")"
 
-if [[ -z "$MANIFEST_URL" ]]; then
+if [[ -z "$MANIFEST_URL" ]] || [[ "$MANIFEST_URL" == "null" ]]; then
    echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]."
    echo "Skipping BC upgrade tests."
    exit 0

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -37,6 +37,7 @@
   <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
   <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIndexingRollingUpgradeIT.java" checks="LineLength" />
   <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]LogsdbIndexingRollingUpgradeIT.java" checks="LineLength" />
+  <suppress files="plugin[/\\]logsdb[/\\]qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]MatchOnlyTextRollingUpgradeIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

docs/changelog/125921.yaml

@@ -0,0 +1,18 @@
+pr: 125921
+summary: Allow direct IO for BBQ rescoring
+area: Vector Search
+type: feature
+highlight:
+  title: Allow direct IO for BBQ rescoring
+  body: |-
+    BBQ rescoring performance can be drastically affected by the amount of available
+    off-heap RAM for use by the system page cache. When there is not enough off-heap RAM
+    to fit all the vector data in memory, BBQ search latencies can be affected by as much as 5000x.
+    Specifying the `vector.rescoring.directio=true` Java option on all vector search
+    nodes modifies rescoring to use direct IO, which eliminates these very high latencies
+    from searches in low-memory scenarios, at a cost of a reduction
+    in vector search performance for BBQ indices when the vectors do all fit in memory.
+
+    This option is released in 9.1 as a tech preview whilst we analyse its effect
+    for a variety of use cases.
+issues: []

docs/changelog/129990.yaml

@@ -0,0 +1,5 @@
+pr: 129990
+summary: Make forecast write load accurate when shard numbers change
+area: Allocation
+type: bug
+issues: []

docs/reference/elasticsearch/rest-apis/api-conventions.md

@@ -421,6 +421,39 @@ GET /_nodes/ra*:2
 GET /_nodes/ra*:2*
 ```
 
+### Component Selectors [api-component-selectors]
+
+A data stream component is a logical grouping of indices that help organize data inside a data stream. All data streams contain a `data` component by default. The `data` component comprises the data stream's backing indices. When searching, managing, or indexing into a data stream, the `data` component is what you are interacting with by default.
+
+Some data stream features are exposed as additional components alongside its `data` component. These other components are comprised of separate sets of backing indices. These additional components store supplemental data independent of the data stream's regular backing indices. An example of another component is the `failures` component exposed by the data stream [failure store](docs-content://manage-data/data-store/data-streams/failure-store.md) feature, which captures documents that fail to be ingested in a separate set of backing indices on the data stream.
+
+Some APIs that accept a `<data-stream>`, `<index>`, or `<target>` request path parameter also support *selector syntax* which defines which component on a data stream the API should operate on. To use a selector, it is appended to the index or data stream name. Selectors can be combined with other index pattern syntax like [date math](#api-date-math-index-names) and wildcards.
+
+There are currently two selector suffixes supported by {{es}} APIs:
+
+`::data`
+:   Refers to a data stream's backing indices containing regular data. Data streams always contain a data component.
+
+`::failures`
+:   This component refers to the internal indices used for a data stream's [failure store](docs-content://manage-data/data-store/data-streams/failure-store.md).
+
+As an example, [search]({{es-apis}}group/endpoint-search), [field capabilities]({{es-apis}}operation/operation-field-caps), and [index stats]({{es-apis}}operation/operation-indices-stats) APIs can all report results from a different component rather than from the default data.
+
+```console
+# Search a data stream normally
+GET my-data-stream/_search
+# Search a data stream's failure data if present
+GET my-data-stream::failures/_search
+
+# Syntax can be combined with other index pattern syntax (wildcards, multi-target, date math, cross cluster search, etc)
+GET logs-*::failures/_search
+GET logs-*::data,logs-*::failures/_count
+GET remote-cluster:logs-*-*::failures/_search
+GET *::data,*::failures,-logs-rdbms-*::failures/_stats
+GET <logs-{now/d}>::failures/_search
+```
+
+
 ## Parameters [api-conventions-parameters]
 
 Rest parameters (when using HTTP, map to HTTP URL parameters) follow the convention of using underscore casing.

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java

@@ -222,9 +222,7 @@ private void updateSettingsOnIndices(
         Map<String, Object> settingsToApply = new HashMap<>();
         List<String> appliedToDataStreamOnly = new ArrayList<>();
         List<String> appliedToDataStreamAndBackingIndices = new ArrayList<>();
-        Settings effectiveSettings = dataStream.getEffectiveSettings(
-            clusterService.state().projectState(projectResolver.getProjectId()).metadata()
-        );
+        Settings effectiveSettings = dataStream.getEffectiveSettings(projectResolver.getProjectMetadata(clusterService.state()));
         for (String settingName : requestSettings.keySet()) {
             if (APPLY_TO_BACKING_INDICES.contains(settingName)) {
                 settingsToApply.put(settingName, effectiveSettings.get(settingName));

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java

@@ -151,7 +151,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @FixForMultiProject(description = "Replace ProjectId.DEFAULT")
     static EnterpriseGeoIpTaskState getEnterpriseGeoIpTaskState(ClusterState state) {
         PersistentTasksCustomMetadata.PersistentTask<?> task = getTaskWithId(
-            state.projectState(ProjectId.DEFAULT).metadata(),
+            state.metadata().getProject(ProjectId.DEFAULT),
             EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER
         );
         return (task == null) ? null : (EnterpriseGeoIpTaskState) task.getState();

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java

@@ -20,7 +20,9 @@
 import org.elasticsearch.cluster.SimpleBatchedExecutor;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
@@ -58,13 +60,15 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) {
     };
 
     private final MasterServiceTaskQueue<UpdateDatabaseConfigurationTask> updateDatabaseConfigurationTaskQueue;
+    private final ProjectResolver projectResolver;
 
     @Inject
     public TransportPutDatabaseConfigurationAction(
         TransportService transportService,
         ClusterService clusterService,
         ThreadPool threadPool,
-        ActionFilters actionFilters
+        ActionFilters actionFilters,
+        ProjectResolver projectResolver
     ) {
         super(
             PutDatabaseConfigurationAction.NAME,
@@ -81,6 +85,7 @@ public TransportPutDatabaseConfigurationAction(
             Priority.NORMAL,
             UPDATE_TASK_EXECUTOR
         );
+        this.projectResolver = projectResolver;
     }
 
     @Override
@@ -89,7 +94,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A
 
         updateDatabaseConfigurationTaskQueue.submitTask(
             Strings.format("update-geoip-database-configuration-[%s]", id),
-            new UpdateDatabaseConfigurationTask(listener, request.getDatabase()),
+            new UpdateDatabaseConfigurationTask(projectResolver.getProjectId(), listener, request.getDatabase()),
             null
         );
     }
@@ -105,9 +110,9 @@ static boolean isNoopUpdate(@Nullable DatabaseConfigurationMetadata existingData
         }
     }
 
-    static void validatePrerequisites(DatabaseConfiguration database, ClusterState state) {
+    static void validatePrerequisites(ProjectId projectId, DatabaseConfiguration database, ClusterState state) {
         // we need to verify that the database represents a unique file (name) among the various databases for this same provider
-        IngestGeoIpMetadata geoIpMeta = state.metadata().getProject().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY);
+        IngestGeoIpMetadata geoIpMeta = state.metadata().getProject(projectId).custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY);
 
         Optional<DatabaseConfiguration> sameName = geoIpMeta.getDatabases()
             .values()
@@ -125,12 +130,14 @@ static void validatePrerequisites(DatabaseConfiguration database, ClusterState s
         });
     }
 
-    private record UpdateDatabaseConfigurationTask(ActionListener<AcknowledgedResponse> listener, DatabaseConfiguration database)
-        implements
-            ClusterStateTaskListener {
+    private record UpdateDatabaseConfigurationTask(
+        ProjectId projectId,
+        ActionListener<AcknowledgedResponse> listener,
+        DatabaseConfiguration database
+    ) implements ClusterStateTaskListener {
 
         ClusterState execute(ClusterState currentState) throws Exception {
-            final var project = currentState.metadata().getProject();
+            final var project = currentState.metadata().getProject(projectId);
             IngestGeoIpMetadata geoIpMeta = project.custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY);
 
             String id = database.id();
@@ -140,7 +147,7 @@ ClusterState execute(ClusterState currentState) throws Exception {
                 return currentState;
             }
 
-            validatePrerequisites(database, currentState);
+            validatePrerequisites(projectId, database, currentState);
 
             Map<String, DatabaseConfigurationMetadata> databases = new HashMap<>(geoIpMeta.getDatabases());
             databases.put(

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java

@@ -10,7 +10,8 @@
 package org.elasticsearch.ingest.geoip.direct;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
 import org.elasticsearch.test.ESTestCase;
 
@@ -20,33 +21,34 @@
 public class TransportPutDatabaseConfigurationActionTests extends ESTestCase {
 
     public void testValidatePrerequisites() {
+        ProjectId projectId = randomProjectIdOrDefault();
         // Test that we reject two configurations with the same database name but different ids:
         String name = randomAlphaOfLengthBetween(1, 50);
         IngestGeoIpMetadata ingestGeoIpMetadata = randomIngestGeoIpMetadata(name);
         ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE)
-            .metadata(Metadata.builder(Metadata.EMPTY_METADATA).putCustom(IngestGeoIpMetadata.TYPE, ingestGeoIpMetadata))
+            .putProjectMetadata(ProjectMetadata.builder(projectId).putCustom(IngestGeoIpMetadata.TYPE, ingestGeoIpMetadata).build())
             .build();
         DatabaseConfiguration databaseConfiguration = randomDatabaseConfiguration(randomIdentifier(), name);
         expectThrows(
             IllegalArgumentException.class,
-            () -> TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfiguration, state)
+            () -> TransportPutDatabaseConfigurationAction.validatePrerequisites(projectId, databaseConfiguration, state)
         );
 
         // Test that we do not reject two configurations with different database names:
         String differentName = randomValueOtherThan(name, () -> randomAlphaOfLengthBetween(1, 50));
         DatabaseConfiguration databaseConfigurationForDifferentName = randomDatabaseConfiguration(randomIdentifier(), differentName);
-        TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfigurationForDifferentName, state);
+        TransportPutDatabaseConfigurationAction.validatePrerequisites(projectId, databaseConfigurationForDifferentName, state);
 
         // Test that we do not reject a configuration if none already exists:
-        TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfiguration, ClusterState.EMPTY_STATE);
+        TransportPutDatabaseConfigurationAction.validatePrerequisites(projectId, databaseConfiguration, ClusterState.EMPTY_STATE);
 
         // Test that we do not reject a configuration if one with the same database name AND id already exists:
         DatabaseConfiguration databaseConfigurationSameNameSameId = ingestGeoIpMetadata.getDatabases()
             .values()
             .iterator()
             .next()
             .database();
-        TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfigurationSameNameSameId, state);
+        TransportPutDatabaseConfigurationAction.validatePrerequisites(projectId, databaseConfigurationSameNameSameId, state);
     }
 
     private IngestGeoIpMetadata randomIngestGeoIpMetadata(String name) {

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -362,10 +362,38 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions,
             return toQuery(query, queryShardContext);
         }
 
+        private static class BytesFromMixedStringsBytesRefBlockLoader extends BlockStoredFieldsReader.StoredFieldsBlockLoader {
+            BytesFromMixedStringsBytesRefBlockLoader(String field) {
+                super(field);
+            }
+
+            @Override
+            public Builder builder(BlockFactory factory, int expectedCount) {
+                return factory.bytesRefs(expectedCount);
+            }
+
+            @Override
+            public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException {
+                return new BlockStoredFieldsReader.Bytes(field) {
+                    private final BytesRef scratch = new BytesRef();
+
+                    @Override
+                    protected BytesRef toBytesRef(Object v) {
+                        if (v instanceof BytesRef b) {
+                            return b;
+                        } else {
+                            assert v instanceof String;
+                            return BlockSourceReader.toBytesRef(scratch, v.toString());
+                        }
+                    }
+                };
+            }
+        }
+
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (textFieldType.isSyntheticSource()) {
-                return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource());
+                return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
             }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             // MatchOnlyText never has norms, so we have to use the field names field
@@ -386,7 +414,12 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                 ) {
                     @Override
                     protected BytesRef storedToBytesRef(Object stored) {
-                        return (BytesRef) stored;
+                        if (stored instanceof BytesRef storedBytes) {
+                            return storedBytes;
+                        } else {
+                            assert stored instanceof String;
+                            return new BytesRef(stored.toString());
+                        }
                     }
                 };
             }
@@ -477,7 +510,12 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
             () -> new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), fieldType().name(), leafName()) {
                 @Override
                 protected void write(XContentBuilder b, Object value) throws IOException {
-                    b.value(((BytesRef) value).utf8ToString());
+                    if (value instanceof BytesRef valueBytes) {
+                        b.value(valueBytes.utf8ToString());
+                    } else {
+                        assert value instanceof String;
+                        b.value(value.toString());
+                    }
                 }
             }
         );