Restore index versions 7 in lucene_snapshot_10 (#113317)

Author: javanna

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java

@@ -102,6 +102,7 @@
 import org.apache.lucene.analysis.util.ElisionFilter;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactory;
@@ -480,9 +481,10 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
             )
         );
         filters.add(PreConfiguredTokenFilter.indexVersion("word_delimiter_graph", false, false, (input, version) -> {
+            boolean adjustOffsets = version.onOrAfter(IndexVersions.V_7_3_0);
             return new WordDelimiterGraphFilter(
                 input,
-                true,
+                adjustOffsets,
                 WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE,
                 WordDelimiterGraphFilter.GENERATE_WORD_PARTS | WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS
                     | WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE | WordDelimiterGraphFilter.SPLIT_ON_NUMERICS
@@ -515,6 +517,7 @@ public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
         // This is already broken with normalization, so backwards compat isn't necessary?
         tokenizers.add(PreConfiguredTokenizer.singleton("lowercase", XLowerCaseTokenizer::new));
         tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new));
+
         return tokenizers;
     }
 }

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java

@@ -338,7 +338,7 @@ public void testShingleFilters() {
         Settings settings = Settings.builder()
             .put(
                 IndexMetadata.SETTING_VERSION_CREATED,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current())
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
             )
             .put("path.home", createTempDir().toString())
             .put("index.analysis.filter.synonyms.type", "synonym")
@@ -392,7 +392,7 @@ public void testPreconfiguredTokenFilters() throws IOException {
         Settings settings = Settings.builder()
             .put(
                 IndexMetadata.SETTING_VERSION_CREATED,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current())
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
             )
             .put("path.home", createTempDir().toString())
             .build();

modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper;
 import org.elasticsearch.index.mapper.DocumentParserContext;
@@ -232,7 +233,9 @@ public Builder(String name, IndexVersion version, boolean ignoreMalformedByDefau
             });
 
             // Set up serialization
-            this.strategy.alwaysSerialize();
+            if (version.onOrAfter(IndexVersions.V_7_0_0)) {
+                this.strategy.alwaysSerialize();
+            }
             // serialize treeLevels if treeLevels is configured, OR if defaults are requested and precision is not configured
             treeLevels.setSerializerCheck((id, ic, v) -> ic || (id && precision.get() == null));
             // serialize precision if precision is configured, OR if defaults are requested and treeLevels is not configured

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.geometry.MultiLine;
 import org.elasticsearch.geometry.MultiPoint;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.legacygeo.mapper.LegacyGeoShapeFieldMapper;
 import org.elasticsearch.legacygeo.parsers.ShapeParser;
@@ -390,7 +391,7 @@ public void testParse3DPolygon() throws IOException, ParseException {
 
         LinearRing shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()]));
         Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null);
-        final IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
         final LegacyGeoShapeFieldMapper mapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).build(
             MapperBuilderContext.root(false, false)
         );

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.geometry.MultiLine;
 import org.elasticsearch.geometry.MultiPoint;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.legacygeo.builders.CoordinatesBuilder;
 import org.elasticsearch.legacygeo.builders.EnvelopeBuilder;
@@ -35,6 +36,7 @@
 import org.elasticsearch.legacygeo.parsers.GeoWKTParser;
 import org.elasticsearch.legacygeo.parsers.ShapeParser;
 import org.elasticsearch.legacygeo.test.RandomShapeGenerator;
+import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -325,6 +327,15 @@ public void testParseMixedDimensionPolyWithHoleStoredZ() throws IOException {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().value(builder.toWKT());
         XContentParser parser = createParser(xContentBuilder);
         parser.nextToken();
+
+        final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
+        final LegacyGeoShapeFieldMapper mapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).build(
+            MapperBuilderContext.root(false, false)
+        );
+
+        // test store z disabled
+        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> ShapeParser.parse(parser, mapperBuilder));
+        assertThat(e, hasToString(containsString("unable to add coordinate to CoordinateBuilder: coordinate dimensions do not match")));
     }
 
     @UpdateForV9
@@ -342,6 +353,14 @@ public void testParsePolyWithStoredZ() throws IOException {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().value(builder.toWKT());
         XContentParser parser = createParser(xContentBuilder);
         parser.nextToken();
+
+        final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
+        final LegacyGeoShapeFieldMapper mapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).build(
+            MapperBuilderContext.root(false, false)
+        );
+
+        ShapeBuilder<?, ?, ?> shapeBuilder = ShapeParser.parse(parser, mapperBuilder);
+        assertEquals(shapeBuilder.numDimensions(), 3);
     }
 
     @UpdateForV9
@@ -353,6 +372,16 @@ public void testParseOpenPolygon() throws IOException {
         XContentParser parser = createParser(xContentBuilder);
         parser.nextToken();
 
+        final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
+        final LegacyGeoShapeFieldMapper defaultMapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).coerce(
+            false
+        ).build(MapperBuilderContext.root(false, false));
+        ElasticsearchParseException exception = expectThrows(
+            ElasticsearchParseException.class,
+            () -> ShapeParser.parse(parser, defaultMapperBuilder)
+        );
+        assertEquals("invalid LinearRing found (coordinates are not closed)", exception.getMessage());
+
         final LegacyGeoShapeFieldMapper coercingMapperBuilder = new LegacyGeoShapeFieldMapper.Builder(
             "test",
             IndexVersion.current(),

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.legacygeo.test.TestLegacyGeoShapeFieldMapperPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
@@ -54,6 +55,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+@SuppressWarnings("deprecation")
 @UpdateForV9
 @AwaitsFix(bugUrl = "this is testing legacy functionality so can likely be removed in 9.0")
 public class LegacyGeoShapeFieldMapperTests extends MapperTestCase {
@@ -125,7 +127,7 @@ protected boolean supportsMeta() {
 
     @Override
     protected IndexVersion getVersion() {
-        return IndexVersions.V_8_0_0;
+        return IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
     }
 
     public void testLegacySwitches() throws IOException {

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.legacygeo.mapper.LegacyGeoShapeFieldMapper.GeoShapeFieldType;
+import org.elasticsearch.test.index.IndexVersionUtils;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,7 +41,7 @@ public void testSetStrategyName() {
     }
 
     public void testFetchSourceValue() throws IOException {
-        IndexVersion version = IndexVersions.V_8_0_0;
+        IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
         MappedFieldType mapper = new LegacyGeoShapeFieldMapper.Builder("field", version, false, true).build(
             MapperBuilderContext.root(false, false)
         ).fieldType();

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -42,7 +42,6 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
@@ -57,6 +56,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.snapshots.mockstore.BlobStoreWrapper;
 import org.elasticsearch.telemetry.Measurement;
 import org.elasticsearch.telemetry.TestTelemetryPlugin;
@@ -426,8 +426,7 @@ public void testEnforcedCooldownPeriod() throws IOException {
                 )
             );
         final BytesReference serialized = BytesReference.bytes(
-            // TODO lucene 10 upgrade, we can probably remove the IndexVersions here once we delete all V7 versions
-            modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), IndexVersions.V_8_0_0)
+            modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), SnapshotsService.OLD_SNAPSHOT_FORMAT)
         );
         if (randomBoolean()) {
             repository.blobStore()

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.repositories.FinalizeSnapshotContext;
@@ -155,12 +156,13 @@ class S3Repository extends MeteredBlobStoreRepository {
 
     /**
      * Artificial delay to introduce after a snapshot finalization or delete has finished so long as the repository is still using the
-     * backwards compatible snapshot format from before V_7_6_0. This delay is necessary so that the eventually consistent
-     * nature of AWS S3 does not randomly result in repository corruption when doing repository operations in rapid succession on a
-     * repository in the old metadata format.
+     * backwards compatible snapshot format from before
+     * {@link org.elasticsearch.snapshots.SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION} ({@link IndexVersions#V_7_6_0}).
+     * This delay is necessary so that the eventually consistent nature of AWS S3 does not randomly result in repository corruption when
+     * doing repository operations in rapid succession on a repository in the old metadata format.
      * This setting should not be adjusted in production when working with an AWS S3 backed repository. Doing so risks the repository
      * becoming silently corrupted. To get rid of this waiting period, either create a new S3 repository or remove all snapshots older than
-     * V_7_6_0 from the repository which will trigger an upgrade of the repository metadata to the new
+     * {@link IndexVersions#V_7_6_0} from the repository which will trigger an upgrade of the repository metadata to the new
      * format and disable the cooldown period.
      */
     static final Setting<TimeValue> COOLDOWN_PERIOD = Setting.timeSetting(
@@ -369,12 +371,14 @@ public void onFailure(Exception e) {
 
     private void logCooldownInfo() {
         logger.info(
-            "Sleeping for [{}] after modifying repository [{}] because it contains snapshots older than version [\"7060099\"]"
+            "Sleeping for [{}] after modifying repository [{}] because it contains snapshots older than version [{}]"
                 + " and therefore is using a backwards compatible metadata format that requires this cooldown period to avoid "
                 + "repository corruption. To get rid of this message and move to the new repository metadata format, either remove "
-                + "all snapshots older than version [\"7060099\"] from the repository or create a new repository at an empty location.",
+                + "all snapshots older than version [{}] from the repository or create a new repository at an empty location.",
             coolDown,
-            metadata.name()
+            metadata.name(),
+            SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION,
+            SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION
         );
     }
 

qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
@@ -181,7 +182,9 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException {
             // 7.12.0+ will try to load RepositoryData during repo creation if verify is true, which is impossible in case of version
             // incompatibility in the downgrade test step. We verify that it is impossible here and then create the repo using verify=false
             // to check behavior on other operations below.
-            final boolean verify = TEST_STEP != TestStep.STEP3_OLD_CLUSTER || SnapshotsService.includesUUIDs(minNodeVersion);
+            final boolean verify = TEST_STEP != TestStep.STEP3_OLD_CLUSTER
+                || SnapshotsService.includesUUIDs(minNodeVersion)
+                || minNodeVersion.before(IndexVersions.V_7_12_0);
             if (verify == false) {
                 expectThrowsAnyOf(EXPECTED_BWC_EXCEPTIONS, () -> createRepository(repoName, false, true));
             }