Merge branch 'main' into merge-disk-space-aware-take-2

Author: albertzaharovits

docs/changelog/128139.yaml

@@ -0,0 +1,5 @@
+pr: 128139
+summary: Skip indexing points for `seq_no` in tsdb and logsdb
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/128163.yaml

@@ -0,0 +1,17 @@
+pr: 128163
+summary: Make `skip_unavailable` catch all errors
+area: ES|QL
+type: breaking
+issues: [ ]
+breaking:
+  title: Cluster setting "skip_unavailable" catches all runtime errors
+  area: ES|QL
+  details: "If `skip_unavailable` is set to `true`, the runtime errors from this cluster\
+    \ do not lead to a failure of the query. Instead, the cluster is set to `skipped`\
+    \ or `partial` status, and the query execution continues. This is a breaking change\
+    \ from previous versions, where `skip_unavailable` only applied to errors related\
+    \ to a cluster being unavailable."
+  impact: "The errors on remote clusters, e.g. missing indices, will not lead to a\
+    \ failure of the query. Instead, the cluster is set to `skipped` or `partial` status\
+    \ in the response metadata."
+  notable: false

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
@@ -53,7 +54,7 @@ public class TransportUpdateDataStreamSettingsAction extends TransportMasterNode
     UpdateDataStreamSettingsAction.Request,
     UpdateDataStreamSettingsAction.Response> {
     private static final Logger logger = LogManager.getLogger(TransportUpdateDataStreamSettingsAction.class);
-    private static final Set<String> APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name");
+    private static final Set<String> APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name", IndexSettings.PREFER_ILM);
     private static final Set<String> APPLY_TO_DATA_STREAM_ONLY = Set.of("index.number_of_shards");
     private final MetadataDataStreamsService metadataDataStreamsService;
     private final MetadataUpdateSettingsService updateSettingsService;

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml

@@ -51,16 +51,20 @@ setup:
         body:
           index:
             number_of_shards: 2
-            lifecycle.name: my-new-policy
+            lifecycle:
+              name: my-new-policy
+              prefer_ilm: true
   - match: { data_streams.0.name: my-data-stream-1 }
   - match: { data_streams.0.applied_to_data_stream: true }
   - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]}
-  - match: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: [index.lifecycle.name] }
+  - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2  }
   - match: { data_streams.0.settings.index.number_of_shards: "2" }
   - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" }
   - match: { data_streams.0.effective_settings.index.number_of_shards: "2" }
   - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" }
   - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" }
 
   - do:
       indices.rollover:
@@ -79,13 +83,15 @@ setup:
   - match: { data_streams.0.effective_settings.index.number_of_shards: "2" }
   - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" }
   - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" }
 
   - do:
       indices.get_data_stream:
         name: my-data-stream-1
   - match: { data_streams.0.name: my-data-stream-1 }
   - match: { data_streams.0.settings.index.number_of_shards: "2" }
   - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" }
   - match: { data_streams.0.effective_settings: null }
 
   - do:
@@ -101,6 +107,7 @@ setup:
   - match: { .$idx0name.settings.index.lifecycle.name: "my-new-policy" }
   - match: { .$idx1name.settings.index.number_of_shards: "2" }
   - match: { .$idx1name.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { .$idx1name.settings.index.lifecycle.prefer_ilm: "true" }
 
 ---
 "Test multiple data streams":

muted-tests.yml

@@ -486,6 +486,11 @@ tests:
 - class: org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderIT
   method: testSearchWhileRelocating
   issue: https://github.com/elastic/elasticsearch/issues/128500
+- class: org.elasticsearch.xpack.esql.ccq.EsqlRestValidationIT
+  issue: https://github.com/elastic/elasticsearch/issues/128543
+- class: org.elasticsearch.xpack.ccr.index.engine.FollowingEngineTests
+  method: testProcessOnceOnPrimary
+  issue: https://github.com/elastic/elasticsearch/issues/128541
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java

@@ -36,6 +36,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.IntStream;
@@ -61,6 +62,13 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return plugins;
     }
 
+    @Override
+    protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) {
+        var b = super.setRandomIndexSettings(random, builder);
+        b.remove(IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING.getKey());
+        return b;
+    }
+
     private static final Set<ShardId> failOnFlushShards = ConcurrentCollections.newConcurrentSet();
 
     public static class EngineTestPlugin extends Plugin implements EnginePlugin {

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -224,6 +224,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 IndexMetadata.INDEX_ROUTING_PATH,
                 IndexSettings.TIME_SERIES_START_TIME,
                 IndexSettings.TIME_SERIES_END_TIME,
+                IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING,
 
                 // Legacy index settings we must keep around for BWC from 7.x
                 EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS,

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.translog.Translog;
@@ -829,6 +830,23 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) {
         }
     }
 
+    public static final Setting<SeqNoFieldMapper.SeqNoIndexOptions> SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting(
+        SeqNoFieldMapper.SeqNoIndexOptions.class,
+        settings -> {
+            final IndexMode indexMode = IndexSettings.MODE.get(settings);
+            if ((indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES)
+                && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.SEQ_NO_WITHOUT_POINTS)) {
+                return SeqNoFieldMapper.SeqNoIndexOptions.DOC_VALUES_ONLY.toString();
+            } else {
+                return SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES.toString();
+            }
+        },
+        "index.seq_no.index_options",
+        value -> {},
+        Property.IndexScope,
+        Property.Final
+    );
+
     private final Index index;
     private final IndexVersion version;
     private final Logger logger;
@@ -933,6 +951,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private volatile int maxRegexLength;
 
     private final IndexRouting indexRouting;
+    private final SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions;
 
     /**
      * The default mode for storing source, for all mappers not overriding this setting.
@@ -1099,6 +1118,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false
             && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
         useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
+        seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
         if (recoverySourceSyntheticEnabled) {
             if (DiscoveryNode.isStateless(settings)) {
                 throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful");
@@ -1837,4 +1857,8 @@ public DenseVectorFieldMapper.FilterHeuristic getHnswFilterHeuristic() {
     private void setHnswFilterHeuristic(DenseVectorFieldMapper.FilterHeuristic heuristic) {
         this.hnswFilterHeuristic = heuristic;
     }
+
+    public SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions() {
+        return seqNoIndexOptions;
+    }
 }

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -168,6 +168,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1);
     public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1);
     public static final IndexVersion DEFAULT_TO_ACORN_HNSW_FILTER_HEURISTIC = def(9_026_0_00, Version.LUCENE_10_2_1);
+    public static final IndexVersion SEQ_NO_WITHOUT_POINTS = def(9_027_0_00, Version.LUCENE_10_2_1);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
@@ -1811,7 +1810,10 @@ private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException
     private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws IOException {
         assert assertMaxSeqNoOfUpdatesIsAdvanced(delete.uid(), delete.seqNo(), false, false);
         try {
-            final ParsedDocument tombstone = ParsedDocument.deleteTombstone(delete.id());
+            final ParsedDocument tombstone = ParsedDocument.deleteTombstone(
+                engineConfig.getIndexSettings().seqNoIndexOptions(),
+                delete.id()
+            );
             assert tombstone.docs().size() == 1 : "Tombstone doc should have single doc [" + tombstone + "]";
             tombstone.updateSeqID(delete.seqNo(), delete.primaryTerm());
             tombstone.version().setLongValue(plan.versionOfDeletion);
@@ -1970,7 +1972,10 @@ private NoOpResult innerNoOp(final NoOp noOp) throws IOException {
                 markSeqNoAsSeen(noOp.seqNo());
                 if (hasBeenProcessedBefore(noOp) == false) {
                     try {
-                        final ParsedDocument tombstone = ParsedDocument.noopTombstone(noOp.reason());
+                        final ParsedDocument tombstone = ParsedDocument.noopTombstone(
+                            engineConfig.getIndexSettings().seqNoIndexOptions(),
+                            noOp.reason()
+                        );
                         tombstone.updateSeqID(noOp.seqNo(), noOp.primaryTerm());
                         // A noop tombstone does not require a _version but it's added to have a fully dense docvalues for the version
                         // field. 1L is selected to optimize the compression because it might probably be the most common value in
@@ -2753,10 +2758,10 @@ private IndexWriterConfig getIndexWriterConfig() {
                 ? SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME
                 : SourceFieldMapper.RECOVERY_SOURCE_NAME,
             engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES,
-            softDeletesPolicy::getRetentionQuery,
+            () -> softDeletesPolicy.getRetentionQuery(engineConfig.getIndexSettings().seqNoIndexOptions()),
             new SoftDeletesRetentionMergePolicy(
                 Lucene.SOFT_DELETES_FIELD,
-                softDeletesPolicy::getRetentionQuery,
+                () -> softDeletesPolicy.getRetentionQuery(engineConfig.getIndexSettings().seqNoIndexOptions()),
                 new PrunePostingsMergePolicy(mergePolicy, IdFieldMapper.NAME)
             )
         );
@@ -3215,12 +3220,7 @@ public int countChanges(String source, long fromSeqNo, long toSeqNo) throws IOEx
         ensureOpen();
         refreshIfNeeded(source, toSeqNo);
         try (Searcher searcher = acquireSearcher(source, SearcherScope.INTERNAL)) {
-            return LuceneChangesSnapshot.countOperations(
-                searcher,
-                fromSeqNo,
-                toSeqNo,
-                config().getIndexSettings().getIndexVersionCreated()
-            );
+            return LuceneChangesSnapshot.countOperations(searcher, engineConfig.getIndexSettings(), fromSeqNo, toSeqNo);
         } catch (Exception e) {
             try {
                 maybeFailEngine("count changes", e);
@@ -3448,7 +3448,11 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead
         final IndexSearcher searcher = new IndexSearcher(directoryReader);
         searcher.setQueryCache(null);
         final Query query = new BooleanQuery.Builder().add(
-            LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, getPersistedLocalCheckpoint() + 1, Long.MAX_VALUE),
+            SeqNoFieldMapper.rangeQueryForSeqNo(
+                engineConfig.getIndexSettings().seqNoIndexOptions(),
+                getPersistedLocalCheckpoint() + 1,
+                Long.MAX_VALUE
+            ),
             BooleanClause.Occur.MUST
         )
             .add(Queries.newNonNestedFilter(indexVersionCreated), BooleanClause.Occur.MUST) // exclude non-root nested documents