Merge branch 'main' into add-RollingUpgradeDeprecatedSettingsIT

cbuescher · web-flow · commit 556304e5b2d8 · 2025-02-13T17:44:36.000+01:00
diff --git a/docs/changelog/122199.yaml b/docs/changelog/122199.yaml
@@ -0,0 +1,5 @@
+pr: 122199
+summary: Fix issues that prevents using search only snapshots for indices that use index sorting. This is includes Logsdb and time series indices.
+area: Logs
+type: bug
+issues: []
diff --git a/docs/changelog/122496.yaml b/docs/changelog/122496.yaml
@@ -0,0 +1,5 @@
+pr: 122496
+summary: Deduplicate `IngestStats` and `IngestStats.Stats` identity records when deserializing
+area: Ingest Node
+type: bug
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -20,6 +20,8 @@
 import org.apache.lucene.index.ConcurrentMergeScheduler;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FilterCodecReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
@@ -190,14 +192,26 @@ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Direc
                 throw new IllegalStateException("no commit found in the directory");
             }
         }
+        // Need to figure out what the parent field is that, so that validation in IndexWriter doesn't fail
+        // if no parent field is configured, but FieldInfo says there is a parent field.
+        String parentField = null;
         final IndexCommit cp = getIndexCommit(si, directory);
+        try (var reader = DirectoryReader.open(cp)) {
+            var topLevelFieldInfos = FieldInfos.getMergedFieldInfos(reader);
+            for (FieldInfo fieldInfo : topLevelFieldInfos) {
+                if (fieldInfo.isParentField()) {
+                    parentField = fieldInfo.getName();
+                }
+            }
+        }
         try (
             IndexWriter writer = new IndexWriter(
                 directory,
                 indexWriterConfigWithNoMerging(Lucene.STANDARD_ANALYZER).setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
                     .setIndexCommit(cp)
                     .setCommitOnClose(false)
                     .setOpenMode(IndexWriterConfig.OpenMode.APPEND)
+                    .setParentField(parentField)
             )
         ) {
             // do nothing and close this will kick off IndexFileDeleter which will remove all pending files
diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java
@@ -232,7 +232,8 @@ public IndexService(
                 mapperMetrics
             );
             this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService);
-            if (indexSettings.getIndexSortConfig().hasIndexSort()) {
+            boolean sourceOnly = Boolean.parseBoolean(indexSettings.getSettings().get("index.source_only"));
+            if (indexSettings.getIndexSortConfig().hasIndexSort() && sourceOnly == false) {
                 // we delay the actual creation of the sort order for this index because the mapping has not been merged yet.
                 // The sort order is validated right after the merge of the mapping later in the process.
                 this.indexSortSupplier = () -> indexSettings.getIndexSortConfig()
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java
@@ -57,22 +57,25 @@ public record IngestStats(Stats totalStats, List<PipelineStat> pipelineStats, Ma
      * Read from a stream.
      */
     public static IngestStats read(StreamInput in) throws IOException {
-        var stats = new Stats(in);
+        var stats = readStats(in);
         var size = in.readVInt();
+        if (stats == Stats.IDENTITY && size == 0) {
+            return IDENTITY;
+        }
         var pipelineStats = new ArrayList<PipelineStat>(size);
         var processorStats = Maps.<String, List<ProcessorStat>>newMapWithExpectedSize(size);
 
         for (var i = 0; i < size; i++) {
             var pipelineId = in.readString();
-            var pipelineStat = new Stats(in);
+            var pipelineStat = readStats(in);
             var byteStat = in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? new ByteStats(in) : new ByteStats(0, 0);
             pipelineStats.add(new PipelineStat(pipelineId, pipelineStat, byteStat));
             int processorsSize = in.readVInt();
             var processorStatsPerPipeline = new ArrayList<ProcessorStat>(processorsSize);
             for (var j = 0; j < processorsSize; j++) {
                 var processorName = in.readString();
                 var processorType = in.readString();
-                var processorStat = new Stats(in);
+                var processorStat = readStats(in);
                 processorStatsPerPipeline.add(new ProcessorStat(processorName, processorType, processorStat));
             }
             processorStats.put(pipelineId, Collections.unmodifiableList(processorStatsPerPipeline));
@@ -167,20 +170,28 @@ static Map<String, List<ProcessorStat>> merge(Map<String, List<ProcessorStat>> f
         return totalsPerPipelineProcessor;
     }
 
+    /**
+     * Read {@link Stats} from a stream.
+     */
+    private static Stats readStats(StreamInput in) throws IOException {
+        long ingestCount = in.readVLong();
+        long ingestTimeInMillis = in.readVLong();
+        long ingestCurrent = in.readVLong();
+        long ingestFailedCount = in.readVLong();
+        if (ingestCount == 0 && ingestTimeInMillis == 0 && ingestCurrent == 0 && ingestFailedCount == 0) {
+            return Stats.IDENTITY;
+        } else {
+            return new Stats(ingestCount, ingestTimeInMillis, ingestCurrent, ingestFailedCount);
+        }
+    }
+
     public record Stats(long ingestCount, long ingestTimeInMillis, long ingestCurrent, long ingestFailedCount)
         implements
             Writeable,
             ToXContentFragment {
 
         public static final Stats IDENTITY = new Stats(0, 0, 0, 0);
 
-        /**
-         * Read from a stream.
-         */
-        public Stats(StreamInput in) throws IOException {
-            this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong());
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(ingestCount);
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java
@@ -19,6 +19,7 @@
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class IngestStatsTests extends ESTestCase {
 
@@ -31,6 +32,11 @@ public void testSerialization() throws IOException {
         assertIngestStats(ingestStats, serializedStats);
     }
 
+    public void testIdentitySerialization() throws IOException {
+        IngestStats serializedStats = serialize(IngestStats.IDENTITY);
+        assertThat(serializedStats, sameInstance(IngestStats.IDENTITY));
+    }
+
     public void testStatsMerge() {
         var first = randomStats();
         var second = randomStats();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java
@@ -9,6 +9,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesSkipIndexType;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -252,7 +253,7 @@ private SegmentCommitInfo syncSegment(
                             false,
                             IndexOptions.NONE,
                             DocValuesType.NONE,
-                            fieldInfo.docValuesSkipIndexType(),
+                            DocValuesSkipIndexType.NONE,
                             -1,
                             fieldInfo.attributes(),
                             0,
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.LegacyFormatNames;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
@@ -95,25 +96,39 @@ private DeprecationIssue oldIndicesCheck(
         IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
         // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
         if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false) && isNotDataStreamIndex(indexMetadata, clusterState)) {
-            return new DeprecationIssue(
-                DeprecationIssue.Level.CRITICAL,
-                "Old index with a compatibility version < 9.0",
-                "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
-                "This index has version: " + currentCompatibilityVersion.toReleaseVersion(),
-                false,
-                meta(indexMetadata, indexToTransformIds)
-            );
+            var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds);
+            if (transforms.isEmpty() == false) {
+                return new DeprecationIssue(
+                    DeprecationIssue.Level.CRITICAL,
+                    "One or more Transforms write to this index with a compatibility version < 9.0",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html"
+                        + "#breaking_90_transform_destination_index",
+                    Strings.format(
+                        "This index was created in version [%s] and requires action before upgrading to 9.0. The following transforms are "
+                            + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare "
+                            + "transforms destination indices for your upgrade.",
+                        currentCompatibilityVersion.toReleaseVersion(),
+                        String.join(", ", transforms)
+                    ),
+                    false,
+                    Map.of("reindex_required", true, "transform_ids", transforms)
+                );
+            } else {
+                return new DeprecationIssue(
+                    DeprecationIssue.Level.CRITICAL,
+                    "Old index with a compatibility version < 9.0",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
+                    "This index has version: " + currentCompatibilityVersion.toReleaseVersion(),
+                    false,
+                    Map.of("reindex_required", true)
+                );
+            }
         }
         return null;
     }
 
-    private Map<String, Object> meta(IndexMetadata indexMetadata, Map<String, List<String>> indexToTransformIds) {
-        var transforms = indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of());
-        if (transforms.isEmpty()) {
-            return Map.of("reindex_required", true);
-        } else {
-            return Map.of("reindex_required", true, "transform_ids", transforms);
-        }
+    private List<String> transformIdsForIndex(IndexMetadata indexMetadata, Map<String, List<String>> indexToTransformIds) {
+        return indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of());
     }
 
     private DeprecationIssue ignoredOldIndicesCheck(
@@ -124,16 +139,35 @@ private DeprecationIssue ignoredOldIndicesCheck(
         IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
         // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
         if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true) && isNotDataStreamIndex(indexMetadata, clusterState)) {
-            return new DeprecationIssue(
-                DeprecationIssue.Level.WARNING,
-                "Old index with a compatibility version < 9.0 Has Been Ignored",
-                "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
-                "This read-only index has version: "
-                    + currentCompatibilityVersion.toReleaseVersion()
-                    + " and will be supported as read-only in 9.0",
-                false,
-                meta(indexMetadata, indexToTransformIds)
-            );
+            var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds);
+            if (transforms.isEmpty() == false) {
+                return new DeprecationIssue(
+                    DeprecationIssue.Level.WARNING,
+                    "One or more Transforms write to this old index with a compatibility version < 9.0",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html"
+                        + "#breaking_90_transform_destination_index",
+                    Strings.format(
+                        "This index was created in version [%s] and will be supported as a read-only index in 9.0. The following "
+                            + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more "
+                            + "about how to handle your transforms destination indices.",
+                        currentCompatibilityVersion.toReleaseVersion(),
+                        String.join(", ", transforms)
+                    ),
+                    false,
+                    Map.of("reindex_required", true, "transform_ids", transforms)
+                );
+            } else {
+                return new DeprecationIssue(
+                    DeprecationIssue.Level.WARNING,
+                    "Old index with a compatibility version < 9.0 has been ignored",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
+                    "This read-only index has version: "
+                        + currentCompatibilityVersion.toReleaseVersion()
+                        + " and will be supported as read-only in 9.0",
+                    false,
+                    Map.of("reindex_required", true)
+                );
+            }
         }
         return null;
     }
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java
diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java
