diff --git a/docs/changelog/119503.yaml b/docs/changelog/119503.yaml new file mode 100644 index 0000000000000..da6697eea7b3a --- /dev/null +++ b/docs/changelog/119503.yaml @@ -0,0 +1,6 @@ +pr: 119503 +summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. +area: Search +type: bug +issues: + - 117042 diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 3ed8fc26ac937..f12935a2bec08 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -17,6 +17,7 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; @@ -26,6 +27,13 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.Version; +import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec; import java.io.IOException; import java.util.ArrayList; @@ -37,55 +45,122 @@ */ public abstract class BWCCodec extends Codec { + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; + private final PostingsFormat postingsFormat; + protected BWCCodec(String name) { super(name); - } - @Override - public NormsFormat normsFormat() { - throw new UnsupportedOperationException(); - } + this.fieldInfosFormat = new FieldInfosFormat() { + final FieldInfosFormat wrappedFormat = originalFieldInfosFormat(); - @Override - public TermVectorsFormat termVectorsFormat() { - throw new UnsupportedOperationException(); - } + @Override + public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) + throws IOException { + return filterFields(wrappedFormat.read(directory, segmentInfo, segmentSuffix, iocontext)); + } - @Override - public KnnVectorsFormat knnVectorsFormat() { - throw new UnsupportedOperationException(); - } + @Override + public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) + throws IOException { + wrappedFormat.write(directory, segmentInfo, segmentSuffix, infos, context); + } + }; + + this.segmentInfosFormat = new SegmentInfoFormat() { + final SegmentInfoFormat wrappedFormat = originalSegmentInfoFormat(); - protected static SegmentInfoFormat wrap(SegmentInfoFormat wrapped) { - return new SegmentInfoFormat() { @Override public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException { - return wrap(wrapped.read(directory, segmentName, segmentID, context)); + return wrap(wrappedFormat.read(directory, segmentName, segmentID, context)); } @Override public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException { - wrapped.write(dir, info, ioContext); + wrappedFormat.write(dir, info, ioContext); } }; - } - protected static FieldInfosFormat wrap(FieldInfosFormat wrapped) { - return new FieldInfosFormat() { + this.postingsFormat = new PerFieldPostingsFormat() { @Override - public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) - throws IOException { - return filterFields(wrapped.read(directory, segmentInfo, segmentSuffix, iocontext)); - } - - @Override - public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) - throws IOException { - wrapped.write(directory, segmentInfo, segmentSuffix, infos, context); + public PostingsFormat getPostingsFormatForField(String field) { + throw new UnsupportedOperationException("Old codecs can't be used for writing"); } }; } + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public final SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * This method is never called in practice, as we rewrite field infos to override the info about which features are present in + * the index. Even if norms are present, field info lies about it. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ + @Override + public final NormsFormat normsFormat() { + throw new UnsupportedOperationException(); + } + + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * This method is never called in practice, as we rewrite field infos to override the info about which features are present in + * the index. Even if term vectors are present, field info lies about it. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ + @Override + public final TermVectorsFormat termVectorsFormat() { + throw new UnsupportedOperationException(); + } + + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * The knn vectors can't be present because it is not supported yet in any of the lucene versions that we support for archive indices. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ + @Override + public final KnnVectorsFormat knnVectorsFormat() { + throw new UnsupportedOperationException(); + } + + /** + * Returns the original {@link SegmentInfoFormat} used by this codec. + * This method should be implemented by subclasses to provide the specific + * {@link SegmentInfoFormat} that this codec is intended to use. + * + * @return the original {@link SegmentInfoFormat} used by this codec + */ + protected abstract SegmentInfoFormat originalSegmentInfoFormat(); + + /** + * Returns the original {@link FieldInfosFormat} used by this codec. + * This method should be implemented by subclasses to provide the specific + * {@link FieldInfosFormat} that this codec is intended to use. + * + * @return the original {@link FieldInfosFormat} used by this codec + */ + protected abstract FieldInfosFormat originalFieldInfosFormat(); + // mark all fields as no term vectors, no norms, no payloads, and no vectors. private static FieldInfos filterFields(FieldInfos fieldInfos) { List fieldInfoCopy = new ArrayList<>(fieldInfos.size()); @@ -118,13 +193,14 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { } public static SegmentInfo wrap(SegmentInfo segmentInfo) { - final Codec codec = segmentInfo.getCodec(); + Codec codec = getBackwardCompatibleCodec(segmentInfo.getCodec()); + final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) // TODO: perhaps store the original version information in attributes so that we can retrieve it later when needed? - org.apache.lucene.util.Version.LATEST, - org.apache.lucene.util.Version.LATEST, + Version.LATEST, + Version.LATEST, segmentInfo.name, segmentInfo.maxDoc(), segmentInfo.getUseCompoundFile(), @@ -139,6 +215,29 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { return segmentInfo1; } + /** + * Returns a backward-compatible codec for the given codec. If the codec is one of the known Lucene 8.x codecs, + * it returns a corresponding read-only backward-compatible codec. Otherwise, it returns the original codec. + * Lucene 8.x codecs are still shipped with the current version of Lucene. + * Earlier codecs we are providing directly they will also be read-only backward-compatible, but they don't require the renaming. + * + * This switch is only for indices created in ES 6.x, later written into in ES 7.x (Lucene 8.x). Indices created + * in ES 7.x can be read directly by ES if marked read-only, without going through archive indices. + */ + @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) + private static Codec getBackwardCompatibleCodec(Codec codec) { + if (codec == null) return null; + + return switch (codec.getClass().getSimpleName()) { + case "Lucene70Codec" -> new BWCLucene70Codec(); + case "Lucene80Codec" -> new BWCLucene80Codec(); + case "Lucene84Codec" -> new BWCLucene84Codec(); + case "Lucene86Codec" -> new BWCLucene86Codec(); + case "Lucene87Codec" -> new BWCLucene87Codec(); + default -> codec; + }; + } + /** * In-memory postings format that shows no postings available. */ diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java index 31973c8dd4e3e..20ef64c87cb91 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java @@ -44,8 +44,7 @@ */ @Deprecated public class Lucene60Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene50SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -68,18 +67,18 @@ public Lucene60Codec() { } @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene50SegmentInfoFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java index fc90a3e14b944..6499f8af72bb2 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java @@ -28,6 +28,7 @@ import java.io.IOException; /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat} * Allows reading metadata only from Lucene 6.0 point format **/ public class Lucene60MetadataOnlyPointsFormat extends PointsFormat { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java index 2e796a04200fe..8a5ca4acd16cb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java @@ -34,7 +34,10 @@ import java.util.HashMap; import java.util.Map; -/** Reads the metadata of point values previously written with Lucene60PointsWriter */ +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsReader} + * Reads the metadata of point values previously written with Lucene60PointsWriter + */ public final class Lucene60MetadataOnlyPointsReader extends PointsReader { final IndexInput dataIn; final SegmentReadState readState; @@ -105,7 +108,7 @@ public Lucene60MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc int fieldNumber = ent.getKey(); long fp = ent.getValue(); dataIn.seek(fp); - PointValues reader = new MetadataOnlyBKDReader(dataIn); + PointValues reader = new MetadataOnlyBKDReader(dataIn, false); readers.put(fieldNumber, reader); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index 43203caf571f1..ab865cba8722e 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -47,7 +47,7 @@ public class MetadataOnlyBKDReader extends PointValues { final int docCount; final int version; - public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { + public MetadataOnlyBKDReader(IndexInput metaIn, boolean isVersionPost86) throws IOException { version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT); final int numDims = metaIn.readVInt(); final int numIndexDims; @@ -85,6 +85,15 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); + + // This code has been introduced to process IndexInput created with Lucene86Codec+. This is not necessary + // in the read-only version for older formats. + if (isVersionPost86) { + metaIn.readVInt(); + metaIn.readLong(); + // The following fields are not used in this class, but we need to read them to advance the pointer + metaIn.readLong(); + } } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java index 02eb75e2437c5..54d36add17159 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java @@ -44,8 +44,7 @@ */ @Deprecated public class Lucene62Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene62SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -68,18 +67,18 @@ public Lucene62Codec() { } @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene62SegmentInfoFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index 5a49a7a415b9c..a06537dd0f820 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -17,11 +17,9 @@ import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; @@ -33,8 +31,6 @@ */ public class BWCLucene70Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -45,12 +41,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { return defaultDVFormat; } }; - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - throw new IllegalStateException("This codec should only be used for reading, not writing"); - } - }; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); // Needed for SPI loading @SuppressWarnings("unused") @@ -64,13 +55,13 @@ protected BWCLucene70Codec(String name) { } @Override - public FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); } @Override @@ -93,13 +84,8 @@ public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - @Override public PointsFormat pointsFormat() { - return new Lucene60MetadataOnlyPointsFormat(); + return pointsFormat; } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java new file mode 100644 index 0000000000000..9537b4e6f7fa0 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene80; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene80.Lucene80Codec} + * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0 + * (Elasticsearch [7.0.0-7.5.2]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene80Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat(); + + private final StoredFieldsFormat storedFieldsFormat; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene80Codec() { + super("BWCLucene80Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public final CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public final PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java new file mode 100644 index 0000000000000..6771f4b3130c1 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene84; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene84.Lucene84Codec} + * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 + * (Elasticsearch [7.6.0-7.8.1]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene84Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final DocValuesFormat defaultDVFormat; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene84Codec() { + super("BWCLucene84Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.defaultDVFormat = new Lucene80DocValuesFormat(); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java new file mode 100644 index 0000000000000..1949285118aed --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86Codec} + * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0-8.6.2 + * (Elasticsearch [7.9.0-7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene86Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + private final DocValuesFormat defaultDVFormat; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene86Codec() { + super("BWCLucene86Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.defaultDVFormat = new Lucene80DocValuesFormat(); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java new file mode 100644 index 0000000000000..f7902c5c9e2a0 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java @@ -0,0 +1,56 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.codecs.PointsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat} + * Allows reading metadata only from Lucene 8.6 point format + **/ +public class Lucene86MetadataOnlyPointsFormat extends PointsFormat { + + static final String META_CODEC_NAME = "Lucene86PointsFormatMeta"; + + /** Filename extension for the meta per field */ + public static final String META_EXTENSION = "kdm"; + + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + + /** Sole constructor */ + public Lucene86MetadataOnlyPointsFormat() {} + + @Override + public PointsWriter fieldsWriter(SegmentWriteState state) { + throw new UnsupportedOperationException("Old codecs may only be used for reading"); + } + + @Override + public PointsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene86MetadataOnlyPointsReader(state); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java new file mode 100644 index 0000000000000..55671828b4dcd --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -0,0 +1,121 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.ChecksumIndexInput; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsReader} + * Reads the metadata of point values previously written with Lucene86PointsWriter + */ +public final class Lucene86MetadataOnlyPointsReader extends PointsReader { + final SegmentReadState readState; + final Map readers = new HashMap<>(); + + public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOException { + this.readState = readState; + + String metaFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.META_EXTENSION + ); + + boolean success = false; + try { + try ( + ChecksumIndexInput metaIn = EndiannessReverserUtil.openChecksumInput(readState.directory, metaFileName, readState.context) + ) { + Throwable priorE = null; + try { + CodecUtil.checkIndexHeader( + metaIn, + Lucene86MetadataOnlyPointsFormat.META_CODEC_NAME, + Lucene86MetadataOnlyPointsFormat.VERSION_START, + Lucene86MetadataOnlyPointsFormat.VERSION_CURRENT, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + + while (true) { + int fieldNumber = metaIn.readInt(); + if (fieldNumber == -1) { + break; + } else if (fieldNumber < 0) { + throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn); + } + PointValues reader = new MetadataOnlyBKDReader(metaIn, true); + readers.put(fieldNumber, reader); + } + metaIn.readLong(); + metaIn.readLong(); + } catch (Throwable t) { + priorE = t; + } finally { + CodecUtil.checkFooter(metaIn, priorE); + } + } + + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public PointValues getValues(String fieldName) { + FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName); + if (fieldInfo == null) { + throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized"); + } + if (fieldInfo.getPointDimensionCount() == 0) { + throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values"); + } + + return readers.get(fieldInfo.number); + } + + // We only open the metadata field, and do nothing with the other two files (index/data), + // for which Lucene checks integrity but we don't need to. + @Override + public void checkIntegrity() {} + + @Override + public void close() throws IOException { + // Free up heap: + readers.clear(); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java new file mode 100644 index 0000000000000..f461bdee8864d --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.Lucene86MetadataOnlyPointsFormat; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene87.Lucene87Codec} + * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.3 + * (Elasticsearch [7.10.0-7-17.26]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene87Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + private final DocValuesFormat defaultDVFormat; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene87Codec() { + super("BWCLucene87Codec"); + this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION); + this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 0215e9f7ca4ab..684902c1bf71f 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,6 +5,10 @@ # 2.0. # +org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java new file mode 100644 index 0000000000000..219cfa29f13ce --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.elasticsearch.test.ESTestCase; + +/** + * Unit tests for the {@link BWCCodec} class. + */ +public class BWCCodecTests extends ESTestCase { + + private final Codec codec; + + public BWCCodecTests() { + this.codec = new BWCCodec("WrapperCodec") { + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return null; + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return null; + } + + @Override + public PostingsFormat postingsFormat() { + return null; + } + + @Override + public DocValuesFormat docValuesFormat() { + return null; + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return null; + } + + @Override + public LiveDocsFormat liveDocsFormat() { + return null; + } + + @Override + public CompoundFormat compoundFormat() { + return null; + } + + @Override + public PointsFormat pointsFormat() { + return null; + } + }; + } + + /** + * Tests that the {@link Codec#normsFormat()} method throws an {@link UnsupportedOperationException}. + */ + public void testNormsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::normsFormat); + } + + /** + * Tests that the {@link Codec#termVectorsFormat()} method throws an {@link UnsupportedOperationException}. + */ + public void testTermVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); + } + + /** + * Tests that the {@link Codec#knnVectorsFormat()} method throws an {@link UnsupportedOperationException}. + */ + public void testKnnVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java index 18cf1b49f5f37..11bca0624eabc 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java @@ -7,18 +7,62 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; +import org.apache.lucene.codecs.Codec; import org.elasticsearch.Version; import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.test.ESTestCase; +import java.util.ServiceLoader; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class OldCodecsAvailableTests extends ESTestCase { /** + * This test verifies for each Lucene codec available via SPI; we also provide a corresponding BWC codec counterpart. + * Using a ServiceLoader, we fetch all classes matching the codecPathRegex (this is applied for Lucne8xCodec at the moment). + * For each entry of the returned list, we intend to load the BWC counterpart reflectively. + * * Reminder to add Lucene BWC codecs under {@link org.elasticsearch.xpack.lucene.bwc.codecs} whenever Elasticsearch is upgraded * to the next major Lucene version. */ @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) public void testLuceneBWCCodecsAvailable() { - assertEquals("Add Lucene BWC codecs for Elasticsearch version 7", 9, Version.CURRENT.major); + assertEquals("Add Lucene BWC codecs for Elasticsearch version 8", 9, Version.CURRENT.major); + + String codecPathRegex = ".*[\\\\.](Lucene(8[0-9])Codec)"; + Pattern codecPathPattern = Pattern.compile(codecPathRegex); + + String codecClassNameRegex = "Lucene(\\d+)Codec"; + Pattern classNamePattern = Pattern.compile(codecClassNameRegex); + + for (Codec codec : ServiceLoader.load(Codec.class)) { + Matcher codecPathMatcher = codecPathPattern.matcher(codec.getClass().getName()); + if (codecPathMatcher.matches()) { + String pathName = codec.getClass().getName(); + int lastDotIndex = pathName.lastIndexOf('.'); + String className = pathName.substring(lastDotIndex + 1); + + Matcher classNameMatcher = classNamePattern.matcher(className); + if (classNameMatcher.matches()) { + String codecVersion = classNameMatcher.group(1); + String wrappedCodecClassPath = "org.elasticsearch.xpack.lucene.bwc.codecs.lucene" + + codecVersion + + ".BWCLucene" + + codecVersion + + "Codec"; + assertTrue(isClassPresent(wrappedCodecClassPath)); + } + } + } + } + + private static boolean isClassPresent(String className) { + try { + Class.forName(className); + return true; + } catch (ClassNotFoundException e) { + return false; + } } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java index be9f98406bc96..e6c1c2571c06f 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java @@ -14,8 +14,11 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -37,6 +40,7 @@ import java.nio.file.Paths; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Consumer; import java.util.stream.Stream; @@ -50,7 +54,7 @@ import static org.hamcrest.Matchers.notNullValue; @TestCaseOrdering(AbstractUpgradeCompatibilityTestCase.TestCaseOrdering.class) -public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase { +public class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase { protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible")); protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible")); @@ -75,17 +79,9 @@ public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCas private static boolean upgradeFailed = false; private final Version clusterVersion; - private final String indexCreatedVersion; - private final Consumer> warningsConsumer; - public AbstractUpgradeCompatibilityTestCase( - @Name("cluster") Version clusterVersion, - String indexCreatedVersion, - Consumer> warningsConsumer - ) { + public AbstractUpgradeCompatibilityTestCase(@Name("cluster") Version clusterVersion) { this.clusterVersion = clusterVersion; - this.indexCreatedVersion = indexCreatedVersion; - this.warningsConsumer = warningsConsumer; } @ParametersFactory @@ -131,7 +127,7 @@ public void maybeUpgrade() throws Exception { assumeFalse("Cluster upgrade failed", upgradeFailed); } - protected static Version clusterVersion() throws Exception { + private static Version clusterVersion() throws Exception { var response = assertOK(client().performRequest(new Request("GET", "/"))); var responseBody = createFromResponse(response); var version = Version.fromString(responseBody.evaluate("version.number").toString()); @@ -151,43 +147,99 @@ public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { } } - protected final void verifyCompatibility(String version, Consumer> warningsConsumer) throws Exception { - final String repository = "repository"; - final String snapshot = "snapshot"; - final String index = "index"; - final int numDocs = 5; + /** + * This method executes in two phases. For cluster in version Current-1, restores and mounts an index snapshot and performs assertions. + * For cluster in version Current, asserts the previously restored/mounted index exists in the upgraded setup. + * + * @param extension The snapshot suffix in resources covering different scenarios. + * @throws Exception + */ + protected final void verifyCompatibility(String extension) throws Exception { + final String repository = "repository_" + extension; + final String index = "index_" + extension; + final String indexMount = index + "_" + repository; + final int numDocs = 1; String repositoryPath = REPOSITORY_PATH.getRoot().getPath(); if (VERSION_MINUS_1.equals(clusterVersion())) { assertEquals(VERSION_MINUS_1, clusterVersion()); - assertTrue(getIndices(client()).isEmpty()); - - // Copy a snapshot of an index with 5 documents - copySnapshotFromResources(repositoryPath, version); - registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build()); - recover(client(), repository, snapshot, index, warningsConsumer); + restoreMountAndAssertSnapshot(extension, repository, repositoryPath, index, indexMount, numDocs, o -> {}); + } + if (VERSION_CURRENT.equals(clusterVersion())) { + assertEquals(VERSION_CURRENT, clusterVersion()); assertTrue(getIndices(client()).contains(index)); assertDocCount(client(), index, numDocs); - return; + assertTrue(getIndices(client()).contains(indexMount)); + assertDocCount(client(), indexMount, numDocs); } + } + + /** + * This method executes in two phases. For cluster in version Current-1 it does not do anything. For cluster in version Current, + * restores and mounts an index snapshot and performs assertions. The test is performed only for the Current cluster. + * + * @param extension The snapshot suffix in resources covering different scenarios. + * @throws Exception + */ + protected final void verifyCompatibilityNoUpgrade(String extension) throws Exception { + verifyCompatibilityNoUpgrade(extension, o -> {}); + } + + protected final void verifyCompatibilityNoUpgrade(String extension, Consumer> warningsConsumer) throws Exception { if (VERSION_CURRENT.equals(clusterVersion())) { + String repository = "repository_" + extension; + String index = "index_" + extension; + String indexMount = index + "_" + repository; + int numDocs = 1; + assertEquals(VERSION_CURRENT, clusterVersion()); - assertTrue(getIndices(client()).contains(index)); - assertDocCount(client(), index, numDocs); + String repositoryPath = REPOSITORY_PATH.getRoot().getPath(); + restoreMountAndAssertSnapshot(extension, repository, repositoryPath, index, indexMount, numDocs, warningsConsumer); } } - protected abstract void recover( - RestClient restClient, + private void restoreMountAndAssertSnapshot( + String extension, String repository, - String snapshot, + String repositoryPath, String index, + String indexMount, + int numDocs, Consumer> warningsConsumer - ) throws Exception; + ) throws Exception { + copySnapshotFromResources(repositoryPath, extension); + registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build()); + + restore(client(), repository, index, warningsConsumer); + mount(client(), repository, index, indexMount, o -> {}); + + assertTrue(getIndices(client()).contains(index)); + assertDocCount(client(), index, numDocs); + assertPhraseQuery(client(), index, "Elasticsearch Doc"); + + assertTrue(getIndices(client()).contains(indexMount)); + } + + private static void assertPhraseQuery(RestClient client, String indexName, String phrase) throws IOException { + var request = new Request("GET", "/" + indexName + "/_search"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + request.setJsonEntity(Strings.format(""" + { + "query": { + "match": { + "content": "%s" + } + } + }""", phrase)); + Response response = client.performRequest(request); + Map map = responseAsMap(response); + int hits = ((List) ((Map) map.get("hits")).get("hits")).size(); + assertEquals("expected 1 documents but it was a different number", 1, hits); + } private static String getIndices(RestClient client) throws IOException { final Request request = new Request("GET", "_cat/indices"); @@ -195,6 +247,38 @@ private static String getIndices(RestClient client) throws IOException { return EntityUtils.toString(response.getEntity()); } + private void restore(RestClient client, String repository, String index, Consumer> warningsConsumer) throws Exception { + var request = new Request("POST", "/_snapshot/" + repository + "/snapshot/_restore"); + request.addParameter("wait_for_completion", "true"); + request.setJsonEntity(Strings.format(""" + { + "indices": "%s", + "include_global_state": false, + "rename_pattern": "(.+)", + "include_aliases": false + }""", index)); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + Response response = client.performRequest(request); + assertOK(response); + warningsConsumer.accept(response.getWarnings()); + } + + private void mount(RestClient client, String repository, String index, String indexMount, Consumer> warningsConsumer) + throws Exception { + var request = new Request("POST", "/_snapshot/" + repository + "/snapshot/_mount"); + request.addParameter("wait_for_completion", "true"); + request.addParameter("storage", "full_copy"); + request.setJsonEntity(Strings.format(""" + { + "index": "%s", + "renamed_index": "%s" + }""", index, indexMount)); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + Response response = client.performRequest(request); + assertOK(response); + warningsConsumer.accept(response.getWarnings()); + } + private static void copySnapshotFromResources(String repositoryPath, String version) throws IOException, URISyntaxException { Path zipFilePath = Paths.get( Objects.requireNonNull(AbstractUpgradeCompatibilityTestCase.class.getClassLoader().getResource("snapshot_v" + version + ".zip")) @@ -225,7 +309,29 @@ private static void unzip(Path zipFilePath, Path outputDir) throws IOException { } } - public final void testArchiveIndex() throws Exception { - verifyCompatibility(indexCreatedVersion, warningsConsumer); + protected static final class TestSnapshotCases { + // Index created in vES_5 - Basic mapping + public static final String ES_VERSION_5 = "5"; + + // Index created in vES_5 - Custom-Analyzer - standard token filter + public static final String ES_VERSION_5_STANDARD_TOKEN_FILTER = "5_standard_token_filter"; + + // Index created in vES_6 - Basic mapping + public static final String ES_VERSION_6 = "6"; + + // Index created in vES_6 - Custom-Analyzer - standard token filter + public static final String ES_VERSION_6_STANDARD_TOKEN_FILTER = "6_standard_token_filter"; + + // Index created in vES_6 - upgraded to vES_7 LuceneCodec80 + public static final String ES_VERSION_6_LUCENE_CODEC_80 = "lucene_80"; + + // Index created in vES_6 - upgraded to vES_7 LuceneCodec84 + public static final String ES_VERSION_6_LUCENE_CODEC_84 = "lucene_84"; + + // Index created in vES_6 - upgraded to vES_7 LuceneCodec86 + public static final String ES_VERSION_6_LUCENE_CODEC_86 = "lucene_86"; + + // Index created in vES_6 - upgraded to vES_7 LuceneCodec87 + public static final String ES_VERSION_6_LUCENE_CODEC_87 = "lucene_87"; } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/OldRepositoriesIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/OldRepositoriesIT.java new file mode 100644 index 0000000000000..067ae6db72845 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/OldRepositoriesIT.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos; + +import org.elasticsearch.test.cluster.util.Version; + +/** + * Test suite for Archive indices backward compatibility with N-2 versions. + * The test suite creates a cluster in the N-1 version, where N is the current version. + * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version. + * Test methods are executed after each upgrade. + * + * For example the test suite creates a cluster of version 8, then restores a snapshot of an index created + * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index + * is successfully restored. + */ +public class OldRepositoriesIT extends AbstractUpgradeCompatibilityTestCase { + + static { + clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); + } + + public OldRepositoriesIT(Version version) { + super(version); + } + + /** + * Test case restoring a snapshot created in ES_v5 - Basic mapping + * 1. Index Created in ES_v5 + * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v5.zip) + * 3. Index Restored to version: Current-1: 8.x + * 4. Cluster upgraded to version Current: 9.x + */ + public void testRestoreMountIndexVersion5() throws Exception { + verifyCompatibility(TestSnapshotCases.ES_VERSION_5); + } + + /** + * Test case mounting a snapshot created in ES_v6 - Basic mapping + * 1. Index Created in ES_v6 + * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v6.zip) + * 3. Index Restored to version: Current-1: 8.x + * 4. Cluster upgraded to version Current: 9.x + */ + public void testRestoreMountIndexVersion6() throws Exception { + verifyCompatibility(TestSnapshotCases.ES_VERSION_6); + } + + /** + * Test case restoring snapshot created in ES_v5 - Custom-analyzer including a standard-token-filter + { + * "settings": { + * "analysis": { + * "analyzer": { + * "custom_analyzer": { + * "type": "custom", + * "tokenizer": "standard", + * "filter": [ + * "standard", + * "lowercase" + * ] + * } + * } + * } + * }, + * "mappings": { + * "my_type": { + * "properties": { + * "content": { + * "type": "text", + * "analyzer": "custom_analyzer" + * } + * } + * } + * } + * } + * 1. Index Created in ES_v5 + * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v5_standard_token_filter.zip) + * 3. Index Restored to version: Current: 9.x + */ + public void testRestoreMountVersion5StandardTokenFilter() throws Exception { + verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> { + assertEquals(1, warnings.size()); + assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); + }); + } + + /** + * Test case restoring snapshot created in ES_v6 - Custom-analyzer including a standard-token-filter + { + * "settings": { + * "analysis": { + * "analyzer": { + * "custom_analyzer": { + * "type": "custom", + * "tokenizer": "standard", + * "filter": [ + * "standard", + * "lowercase" + * ] + * } + * } + * } + * }, + * "mappings": { + * "my_type": { + * "properties": { + * "content": { + * "type": "text", + * "analyzer": "custom_analyzer" + * } + * } + * } + * } + * } + * 1. Index Created in ES_v6 + * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v6_standard_token_filter.zip) + * 3. Index Restored to version: Current: 9.x + */ + public void testRestoreMountVersion6StandardTokenFilter() throws Exception { + verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> { + assertEquals(1, warnings.size()); + assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); + }); + } + + /** + * Test case restoring snapshot created with luceneCoded8.0 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.0.0 -> LuceneVersion 8.0.0 -> LuceneCodec lucene80 + * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_80.zip) + * 4. Index Restored to version: Current : 9.x + */ + public void testRestoreMountVersion6LuceneCodec80() throws Exception { + verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_80); + } + + /** + * Test case restoring snapshot created with luceneCoded8.4 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.6.0 -> LuceneVersion 8.4.0 -> LuceneCodec lucene84 + * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_84.zip) + * 4. Index Restored to version: Current: 9.x + */ + public void testRestoreMountVersion6LuceneCodec84() throws Exception { + verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_84); + } + + /** + * Test case restoring snapshot created with luceneCoded8.4 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.9.0 -> LuceneVersion 8.6.0 -> LuceneCodec lucene86 + * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_86.zip) + * 4. Index Restored to version: Current: 9.x + */ + public void testRestoreMountVersion6LuceneCodec86() throws Exception { + verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_86); + } + + /** + * Test case restoring snapshot created with luceneCoded8.4 + * 1. Index Created in ES_v6 + * 2. Cluster upgraded to ES_v7.10.0 -> LuceneVersion 8.7.0 -> LuceneCodec lucene87 + * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_87.zip) + * 4. Index Restored to version: Current: 9.x + */ + public void testRestoreMountVersion6LuceneCodec87() throws Exception { + verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_87); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/TestSnapshotCases.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/TestSnapshotCases.java deleted file mode 100644 index da1984c482670..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/TestSnapshotCases.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos; - -/** -* Snapshot zip file names for archive and searchable snapshot tests -*/ -public final class TestSnapshotCases { - // Index created in vES_5 - Basic mapping - public static final String ES_VERSION_5 = "5"; - - // Index created in vES_5 - Custom-Analyzer - standard token filter - public static final String ES_VERSION_5_STANDARD_TOKEN_FILTER = "5_custom_analyzer"; - - // Index created in vES_5 - Basic mapping - public static final String ES_VERSION_6 = "5"; - - // Index created in vES_5 - Custom-Analyzer - standard token filter - public static final String ES_VERSION_6_STANDARD_TOKEN_FILTER = "5_custom_analyzer"; - -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java deleted file mode 100644 index d3704937fd912..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.archiveindex; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.WarningsHandler; -import org.elasticsearch.common.Strings; -import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase; -import org.elasticsearch.test.cluster.util.Version; - -import java.util.List; -import java.util.function.Consumer; - -/** - * Test suite for Archive indices backward compatibility with N-2 versions. - * The test suite creates a cluster in the N-1 version, where N is the current version. - * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version. - * Test methods are executed after each upgrade. - * - * For example the test suite creates a cluster of version 8, then restores a snapshot of an index created - * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index - * is successfully restored. - */ -abstract class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase { - - static { - clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); - } - - protected ArchiveIndexTestCase(Version version, String indexCreatedVersion) { - this(version, indexCreatedVersion, o -> {}); - } - - protected ArchiveIndexTestCase(Version version, String indexCreatedVersion, Consumer> consumer) { - super(version, indexCreatedVersion, consumer); - } - - /** - * Overrides the snapshot-restore operation for archive-indices scenario. - */ - @Override - public void recover(RestClient client, String repository, String snapshot, String index, Consumer> warningsConsumer) - throws Exception { - var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore"); - request.addParameter("wait_for_completion", "true"); - request.setJsonEntity(Strings.format(""" - { - "indices": "%s", - "include_global_state": false, - "rename_pattern": "(.+)", - "include_aliases": false - }""", index)); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - Response response = client.performRequest(request); - assertOK(response); - warningsConsumer.accept(response.getWarnings()); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5CustomAnalyzerIT.java deleted file mode 100644 index df5826fc6b725..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5CustomAnalyzerIT.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.archiveindex; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case restoring index created in ES_v5 - Custom-Analyzer - standard token filter - * - * PUT /index - * { - * "settings": { - * "analysis": { - * "analyzer": { - * "custom_analyzer": { - * "type": "custom", - * "tokenizer": "standard", - * "filter": [ - * "standard", - * "lowercase" - * ] - * } - * } - * } - * }, - * "mappings": { - * "my_type": { - * "properties": { - * "content": { - * "type": "text", - * "analyzer": "custom_analyzer" - * } - * } - * } - * } - * } - */ -public class RestoreFromVersion5CustomAnalyzerIT extends ArchiveIndexTestCase { - - public RestoreFromVersion5CustomAnalyzerIT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> { - assertEquals(1, warnings.size()); - assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); - }); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java deleted file mode 100644 index ca482e54db6bb..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.archiveindex; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case restoring snapshot created in ES_v5 - Basic mapping - * - * PUT /index - * { - * "settings": { - * "number_of_shards": 1, - * "number_of_replicas": 1 - * }, - * "mappings": { - * "my_type": { - * "properties": { - * "title": { - * "type": "text" - * }, - * "created_at": { - * "type": "date" - * }, - * "views": { - * "type": "integer" - * } - * } - * } - * } - * } - */ -public class RestoreFromVersion5IT extends ArchiveIndexTestCase { - - public RestoreFromVersion5IT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_5); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6CustomAnalyzerIT.java deleted file mode 100644 index 63ab202798d2f..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6CustomAnalyzerIT.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.archiveindex; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case restoring index created in ES_v6 - Custom-Analyzer - standard token filter - * - * PUT /index - * { - * "settings": { - * "analysis": { - * "analyzer": { - * "custom_analyzer": { - * "type": "custom", - * "tokenizer": "standard", - * "filter": [ - * "standard", - * "lowercase" - * ] - * } - * } - * } - * }, - * "mappings": { - * "_doc": { - * "properties": { - * "content": { - * "type": "text", - * "analyzer": "custom_analyzer" - * } - * } - * } - * } - * } - */ -public class RestoreFromVersion6CustomAnalyzerIT extends ArchiveIndexTestCase { - - public RestoreFromVersion6CustomAnalyzerIT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> { - assertEquals(1, warnings.size()); - assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); - }); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java deleted file mode 100644 index 843f4dd96297e..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.archiveindex; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case restoring snapshot created in ES_v6 - Basic mapping - * - * PUT /index - * { - * "settings": { - * "number_of_shards": 1, - * "number_of_replicas": 1 - * }, - * "mappings": { - * "_doc": { - * "properties": { - * "title": { - * "type": "text" - * }, - * "content": { - * "type": "text" - * }, - * "created_at": { - * "type": "date" - * } - * } - * } - * } - * } - */ -public class RestoreFromVersion6IT extends ArchiveIndexTestCase { - - public RestoreFromVersion6IT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_6); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5CustomAnalyzerIT.java deleted file mode 100644 index 4440629729ef9..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5CustomAnalyzerIT.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.searchablesnapshot; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case mounting index created in ES_v5 - Custom-Analyzer - standard token filter - * - * PUT /index - * { - * "settings": { - * "analysis": { - * "analyzer": { - * "custom_analyzer": { - * "type": "custom", - * "tokenizer": "standard", - * "filter": [ - * "standard", - * "lowercase" - * ] - * } - * } - * } - * }, - * "mappings": { - * "my_type": { - * "properties": { - * "content": { - * "type": "text", - * "analyzer": "custom_analyzer" - * } - * } - * } - * } - * } - */ -public class MountFromVersion5CustomAnalyzerIT extends SearchableSnapshotTestCase { - - public MountFromVersion5CustomAnalyzerIT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> { - assertEquals(1, warnings.size()); - assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); - }); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java deleted file mode 100644 index d5231be121282..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.searchablesnapshot; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case mounting snapshot created in ES_v5 - Basic mapping - * - * PUT /index - * { - * "settings": { - * "number_of_shards": 1, - * "number_of_replicas": 1 - * }, - * "mappings": { - * "my_type": { - * "properties": { - * "title": { - * "type": "text" - * }, - * "created_at": { - * "type": "date" - * }, - * "views": { - * "type": "integer" - * } - * } - * } - * } - * } - */ -public class MountFromVersion5IT extends SearchableSnapshotTestCase { - - public MountFromVersion5IT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_5); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6CustomAnalyzerIT.java deleted file mode 100644 index de529c95bd72d..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6CustomAnalyzerIT.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.searchablesnapshot; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case mounting index created in ES_v6 - Custom-Analyzer - standard token filter - * - * PUT /index - * { - * "settings": { - * "analysis": { - * "analyzer": { - * "custom_analyzer": { - * "type": "custom", - * "tokenizer": "standard", - * "filter": [ - * "standard", - * "lowercase" - * ] - * } - * } - * } - * }, - * "mappings": { - * "_doc": { - * "properties": { - * "content": { - * "type": "text", - * "analyzer": "custom_analyzer" - * } - * } - * } - * } - * } - */ -public class MountFromVersion6CustomAnalyzerIT extends SearchableSnapshotTestCase { - - public MountFromVersion6CustomAnalyzerIT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> { - assertEquals(1, warnings.size()); - assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst()); - }); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java deleted file mode 100644 index 9632fc0e279b8..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.searchablesnapshot; - -import org.elasticsearch.oldrepos.TestSnapshotCases; -import org.elasticsearch.test.cluster.util.Version; - -/** - * Test case mounting snapshot created in ES_v6 - Basic mapping - * - * PUT /index - * { - * "settings": { - * "number_of_shards": 1, - * "number_of_replicas": 1 - * }, - * "mappings": { - * "_doc": { - * "properties": { - * "title": { - * "type": "text" - * }, - * "content": { - * "type": "text" - * }, - * "created_at": { - * "type": "date" - * } - * } - * } - * } - * } - */ -public class MountFromVersion6IT extends SearchableSnapshotTestCase { - - public MountFromVersion6IT(Version version) { - super(version, TestSnapshotCases.ES_VERSION_6); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java deleted file mode 100644 index 88d0f3901ca1c..0000000000000 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.oldrepos.searchablesnapshot; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.WarningsHandler; -import org.elasticsearch.common.Strings; -import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase; -import org.elasticsearch.test.cluster.util.Version; - -import java.util.List; -import java.util.function.Consumer; - -/** - * Test suite for Searchable indices backward compatibility with N-2 versions. - * The test suite creates a cluster in the N-1 version, where N is the current version. - * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version. - * Test methods are executed after each upgrade. - */ -abstract class SearchableSnapshotTestCase extends AbstractUpgradeCompatibilityTestCase { - - static { - clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); - } - - protected SearchableSnapshotTestCase(Version version, String indexCreatedVersion) { - this(version, indexCreatedVersion, o -> {}); - } - - protected SearchableSnapshotTestCase(Version version, String indexCreatedVersion, Consumer> warningsConsumer) { - super(version, indexCreatedVersion, warningsConsumer); - } - - /** - * Overrides the snapshot-restore operation for archive-indices scenario. - */ - @Override - public void recover(RestClient client, String repository, String snapshot, String index, Consumer> warningsConsumer) - throws Exception { - var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount"); - request.addParameter("wait_for_completion", "true"); - request.addParameter("storage", "full_copy"); - request.setJsonEntity(Strings.format(""" - { - "index": "%s", - "renamed_index": "%s" - }""", index, index)); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); - Response response = client.performRequest(request); - assertOK(response); - warningsConsumer.accept(response.getWarnings()); - } -} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md index 5b3ff1f296414..cae9e7a100b16 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md @@ -124,7 +124,7 @@ PUT /index } ``` -### Create Index Version 5 - Custom-Analyzer +### Create Index Version 6 - Custom-Analyzer ``` PUT /index { diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip index 54dcf4f6182cc..2c82bcb18d9d8 100644 Binary files a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_custom_analyzer.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_custom_analyzer.zip deleted file mode 100644 index cd8a28a825b89..0000000000000 Binary files a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_custom_analyzer.zip and /dev/null differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_standard_token_filter.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_standard_token_filter.zip new file mode 100644 index 0000000000000..479316864876f Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_standard_token_filter.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip index d83152fb71c62..ad191df4547ef 100644 Binary files a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_custom_analyzer.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_custom_analyzer.zip deleted file mode 100644 index 012de2cd5d5dd..0000000000000 Binary files a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_custom_analyzer.zip and /dev/null differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_standard_token_filter.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_standard_token_filter.zip new file mode 100644 index 0000000000000..7e03cbe1f2526 Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_standard_token_filter.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_80.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_80.zip new file mode 100644 index 0000000000000..8cbe09c0337a7 Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_80.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_84.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_84.zip new file mode 100644 index 0000000000000..fd9c879bf7754 Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_84.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_86.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_86.zip new file mode 100644 index 0000000000000..00efa0511f574 Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_86.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_87.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_87.zip new file mode 100644 index 0000000000000..8fec98410df65 Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_vlucene_87.zip differ